diff --git a/datatypes/Cargo.toml b/datatypes/Cargo.toml index f824b51a1..8f938985a 100644 --- a/datatypes/Cargo.toml +++ b/datatypes/Cargo.toml @@ -13,7 +13,8 @@ postgres = ["postgres-types"] [dependencies] # simd compiles with nightly only, allow default features to build with arrow-flight once it is possible -arrow = {version = "2.0", features = ["simd"] } +# arrow = { version = "3.0", features = ["simd"] } TODO: activate when version is patched +arrow = { version = "3.0" } chrono = "0.4" float-cmp = "0.8" geo = "0.17" diff --git a/datatypes/src/collections/batch_builder.rs b/datatypes/src/collections/batch_builder.rs index f55ad7c79..785338814 100644 --- a/datatypes/src/collections/batch_builder.rs +++ b/datatypes/src/collections/batch_builder.rs @@ -226,23 +226,21 @@ impl RawFeatureCollectionBuilder { .len(num_features) .add_buffer(feature_offsets) .add_child_data( - ArrayData::builder(arrow::datatypes::DataType::List( - Coordinate2D::arrow_data_type().into(), - )) - .len(num_lines) - .add_buffer(line_offsets) - .add_child_data( - ArrayData::builder(Coordinate2D::arrow_data_type()) - .len(num_coords) - .add_child_data( - ArrayData::builder(DataType::Float64) - .len(num_floats) - .add_buffer(coords) - .build(), - ) - .build(), - ) - .build(), + ArrayData::builder(Coordinate2D::arrow_list_data_type()) + .len(num_lines) + .add_buffer(line_offsets) + .add_child_data( + ArrayData::builder(Coordinate2D::arrow_data_type()) + .len(num_coords) + .add_child_data( + ArrayData::builder(DataType::Float64) + .len(num_floats) + .add_buffer(coords) + .build(), + ) + .build(), + ) + .build(), ) .build(); @@ -269,31 +267,27 @@ impl RawFeatureCollectionBuilder { .len(num_features) .add_buffer(feature_offsets) .add_child_data( - ArrayData::builder(arrow::datatypes::DataType::List( - arrow::datatypes::DataType::List(Coordinate2D::arrow_data_type().into()).into(), - )) - .len(num_polygons) - .add_buffer(polygon_offsets) - .add_child_data( - ArrayData::builder(arrow::datatypes::DataType::List( - Coordinate2D::arrow_data_type().into(), - )) - .len(num_rings) - .add_buffer(ring_offsets) + ArrayData::builder(MultiLineString::arrow_data_type()) + .len(num_polygons) + .add_buffer(polygon_offsets) .add_child_data( - ArrayData::builder(Coordinate2D::arrow_data_type()) - .len(num_coords) + ArrayData::builder(Coordinate2D::arrow_list_data_type()) + .len(num_rings) + .add_buffer(ring_offsets) .add_child_data( - ArrayData::builder(DataType::Float64) - .len(num_floats) - .add_buffer(coords) + ArrayData::builder(Coordinate2D::arrow_data_type()) + .len(num_coords) + .add_child_data( + ArrayData::builder(DataType::Float64) + .len(num_floats) + .add_buffer(coords) + .build(), + ) .build(), ) .build(), ) .build(), - ) - .build(), ) .build(); @@ -462,7 +456,7 @@ mod tests { let num_bytes = bit_util::ceil(numbers.len(), 8); let mut null_buffer = MutableBuffer::new(num_bytes).with_bitset(num_bytes, false); - let null_slice = null_buffer.data_mut(); + let null_slice = null_buffer.as_slice_mut(); for (i, null) in nulls.iter().enumerate() { if *null { @@ -472,7 +466,7 @@ mod tests { // nulls builder - .set_column::("foo", value_buffer, Some(null_buffer.freeze())) + .set_column::("foo", value_buffer, Some(null_buffer.into())) .unwrap(); builder.finish().unwrap(); diff --git a/datatypes/src/collections/feature_collection.rs b/datatypes/src/collections/feature_collection.rs index 3a8f0881c..971905b6c 100644 --- a/datatypes/src/collections/feature_collection.rs +++ b/datatypes/src/collections/feature_collection.rs @@ -1,6 +1,6 @@ use arrow::array::{ - as_primitive_array, as_string_array, Array, ArrayData, ArrayRef, BooleanArray, Float64Array, - ListArray, PrimitiveArrayOps, StructArray, + as_primitive_array, as_string_array, Array, ArrayData, ArrayRef, BooleanArray, ListArray, + StructArray, }; use arrow::datatypes::{DataType, Field, Float64Type, Int64Type}; use arrow::error::ArrowError; @@ -498,7 +498,7 @@ where downcast_array(array_a), downcast_array(array_b), )?), - _ => arrow::compute::concat(&[array_a.clone(), array_b.clone()])?, + _ => arrow::compute::concat(&[array_a.as_ref(), array_b.as_ref()])?, }, )); } @@ -833,7 +833,7 @@ where unsafe { std::slice::from_raw_parts( - timestamps.raw_values().cast::(), + timestamps.values().as_ptr().cast::(), number_of_time_intervals, ) } @@ -1021,62 +1021,11 @@ impl Clone for FeatureCollection { } } -impl VectorDataTyped for FeatureCollection -where - CollectionType: Geometry, -{ - fn vector_data_type(&self) -> VectorDataType { - CollectionType::DATA_TYPE - } -} - impl PartialEq for FeatureCollection where CollectionType: Geometry + ArrowTyped, { - #[allow(clippy::too_many_lines)] // TODO: split function fn eq(&self, other: &Self) -> bool { - /// compares two `f64` typed columns - /// treats `f64::NAN` values as if they are equal - fn f64_column_equals(a: &Float64Array, b: &Float64Array) -> bool { - if (a.len() != b.len()) || (a.null_count() != b.null_count()) { - return false; - } - let number_of_values = a.len(); - - if a.null_count() == 0 { - let a_values: &[f64] = a.value_slice(0, number_of_values); - let b_values: &[f64] = a.value_slice(0, number_of_values); - - for (&v1, &v2) in a_values.iter().zip(b_values) { - match (v1.is_nan(), v2.is_nan()) { - (true, true) => continue, - (false, false) if float_cmp::approx_eq!(f64, v1, v2) => continue, - _ => return false, - } - } - } else { - for i in 0..number_of_values { - match (a.is_null(i), b.is_null(i)) { - (true, true) => continue, - (false, false) => (), // need to compare values - _ => return false, - }; - - let v1: f64 = a.value(i); - let v2: f64 = b.value(i); - - match (v1.is_nan(), v2.is_nan()) { - (true, true) => continue, - (false, false) if float_cmp::approx_eq!(f64, v1, v2) => continue, - _ => return false, - } - } - } - - true - } - if self.types != other.types { return false; } @@ -1091,122 +1040,8 @@ where let c1 = self.table.column_by_name(key).expect("column must exist"); let c2 = other.table.column_by_name(key).expect("column must exist"); - match (c1.data_type(), c2.data_type()) { - (DataType::Float64, DataType::Float64) => { - if !f64_column_equals(downcast_array(c1), downcast_array(c2)) { - return false; - } - } - (DataType::List(_), DataType::List(_)) => { - // TODO: remove special treatment for geometry types on next arrow version - - match CollectionType::DATA_TYPE { - VectorDataType::Data => {} - VectorDataType::MultiPoint => { - if !c1.equals(c2.as_ref()) { - return false; - } - } - VectorDataType::MultiLineString => { - let c1_feature_offsets = c1.data(); - let c2_feature_offsets = c2.data(); - let c1_lines_offsets = c1_feature_offsets.child_data().first().unwrap(); - let c2_lines_offsets = c2_feature_offsets.child_data().first().unwrap(); - let c1_coordinates = c1_lines_offsets - .child_data() - .first() - .unwrap() - .child_data() - .first() - .unwrap(); - let c2_coordinates = c2_lines_offsets - .child_data() - .first() - .unwrap() - .child_data() - .first() - .unwrap(); - - let feature_offsets_eq = || { - c1_feature_offsets.buffers()[0].data() - == c2_feature_offsets.buffers()[0].data() - }; - - let lines_offsets_eq = || { - c1_lines_offsets.buffers()[0].data() - == c2_lines_offsets.buffers()[0].data() - }; - - let coordinates_eq = || { - c1_coordinates.buffers()[0].data() - == c2_coordinates.buffers()[0].data() - }; - - if !feature_offsets_eq() || !lines_offsets_eq() || !coordinates_eq() { - return false; - } - } - VectorDataType::MultiPolygon => { - let c1_feature_offsets = c1.data(); - let c2_feature_offsets = c2.data(); - let c1_polygons_offsets = - c1_feature_offsets.child_data().first().unwrap(); - let c2_polygons_offsets = - c2_feature_offsets.child_data().first().unwrap(); - let c1_rings_offsets = - c1_polygons_offsets.child_data().first().unwrap(); - let c2_rings_offsets = - c2_polygons_offsets.child_data().first().unwrap(); - let c1_coordinates = c1_rings_offsets - .child_data() - .first() - .unwrap() - .child_data() - .first() - .unwrap(); - let c2_coordinates = c2_rings_offsets - .child_data() - .first() - .unwrap() - .child_data() - .first() - .unwrap(); - - let feature_offsets_eq = || { - c1_feature_offsets.buffers()[0].data() - == c2_feature_offsets.buffers()[0].data() - }; - - let polygons_offsets_eq = || { - c1_polygons_offsets.buffers()[0].data() - == c2_polygons_offsets.buffers()[0].data() - }; - - let rings_offsets_eq = || { - c1_rings_offsets.buffers()[0].data() - == c2_rings_offsets.buffers()[0].data() - }; - - let coordinates_eq = || { - c1_coordinates.buffers()[0].data() - == c2_coordinates.buffers()[0].data() - }; - - if !feature_offsets_eq() - || !polygons_offsets_eq() - || !rings_offsets_eq() - || !coordinates_eq() - { - return false; - } - } - } - } - _ => { - if !c1.equals(c2.as_ref()) { - return false; - } - } + if c1 != c2 { + return false; } } @@ -1214,6 +1049,15 @@ where } } +impl VectorDataTyped for FeatureCollection +where + CollectionType: Geometry, +{ + fn vector_data_type(&self) -> VectorDataType { + CollectionType::DATA_TYPE + } +} + /// This implements `IntoGeometryOptionsIterator` for `FeatureCollection`s that implement `IntoGeometryIterator` impl<'i, CollectionType> IntoGeometryOptionsIterator<'i> for FeatureCollection where @@ -1484,7 +1328,6 @@ mod tests { } #[test] - #[should_panic = "duplicate column"] fn rename_columns_fails() { let collection = DataCollection::from_data( vec![], @@ -1499,8 +1342,8 @@ mod tests { ) .unwrap(); - collection + assert!(collection .rename_columns(&[("foo", "baz"), ("bar", "baz")]) - .expect("duplicate column"); + .is_err()); } } diff --git a/datatypes/src/collections/feature_collection_builder.rs b/datatypes/src/collections/feature_collection_builder.rs index 03316d877..697f6ec2a 100644 --- a/datatypes/src/collections/feature_collection_builder.rs +++ b/datatypes/src/collections/feature_collection_builder.rs @@ -253,13 +253,18 @@ where .builders .values() .map(|builder| { - let data_type_size = match builder.data_type() { - arrow::datatypes::DataType::Float64 => std::mem::size_of::(), - arrow::datatypes::DataType::Int64 => std::mem::size_of::(), - arrow::datatypes::DataType::UInt8 => std::mem::size_of::(), - arrow::datatypes::DataType::Utf8 => 0, // TODO: how to get this dynamic value - _ => unreachable!("This type is not an attribute type"), + let data_type_size = if builder.as_any().is::() { + std::mem::size_of::() + } else if builder.as_any().is::() { + std::mem::size_of::() + } else if builder.as_any().is::() { + std::mem::size_of::() + } else if builder.as_any().is::() { + 0 // TODO: how to get this dynamic value + } else { + unreachable!("This type is not an attribute type"); }; + let values_size = builder.len() * data_type_size; let null_size_estimate = builder.len() / 8; diff --git a/datatypes/src/collections/multi_line_string_collection.rs b/datatypes/src/collections/multi_line_string_collection.rs index 955ad1cbe..6c1056b94 100644 --- a/datatypes/src/collections/multi_line_string_collection.rs +++ b/datatypes/src/collections/multi_line_string_collection.rs @@ -32,7 +32,7 @@ impl GeometryCollection for MultiLineStringCollection { unsafe { slice::from_raw_parts( - floats.raw_values().cast::(), + floats.values().as_ptr().cast::(), number_of_coordinates, ) } @@ -49,7 +49,7 @@ impl GeometryCollection for MultiLineStringCollection { let data = geometries.data(); let buffer = &data.buffers()[0]; - unsafe { slice::from_raw_parts(buffer.raw_data().cast::(), geometries.len() + 1) } + unsafe { slice::from_raw_parts(buffer.as_ptr().cast::(), geometries.len() + 1) } } } @@ -68,7 +68,7 @@ impl MultiLineStringCollection { let data = line_strings.data(); let buffer = &data.buffers()[0]; - unsafe { slice::from_raw_parts(buffer.raw_data().cast::(), line_strings.len() + 1) } + unsafe { slice::from_raw_parts(buffer.as_ptr().cast::(), line_strings.len() + 1) } } } @@ -131,7 +131,7 @@ impl<'l> Iterator for MultiLineStringIterator<'l> { line_coordinate_slices.push(unsafe { #[allow(clippy::cast_ptr_alignment)] slice::from_raw_parts( - float_array.raw_values().cast::(), + float_array.values().as_ptr().cast::(), number_of_coordinates, ) }); @@ -185,7 +185,7 @@ impl<'l> GeometryRandomAccess<'l> for MultiLineStringCollection { line_coordinate_slices.push(unsafe { #[allow(clippy::cast_ptr_alignment)] slice::from_raw_parts( - float_array.raw_values().cast::(), + float_array.values().as_ptr().cast::(), number_of_coordinates, ) }); diff --git a/datatypes/src/collections/multi_point_collection.rs b/datatypes/src/collections/multi_point_collection.rs index 04b15b90c..8313864cf 100755 --- a/datatypes/src/collections/multi_point_collection.rs +++ b/datatypes/src/collections/multi_point_collection.rs @@ -62,7 +62,10 @@ impl<'l> Iterator for MultiPointIterator<'l> { let floats: &Float64Array = downcast_array(&floats_ref); let multi_point = MultiPointRef::new_unchecked(unsafe { - slice::from_raw_parts(floats.raw_values().cast::(), number_of_points) + slice::from_raw_parts( + floats.values().as_ptr().cast::(), + number_of_points, + ) }); self.index += 1; // increment! @@ -104,7 +107,10 @@ impl<'l> GeometryRandomAccess<'l> for MultiPointCollection { let floats: &Float64Array = downcast_array(&floats_ref); let multi_point = MultiPointRef::new_unchecked(unsafe { - slice::from_raw_parts(floats.raw_values().cast::(), number_of_points) + slice::from_raw_parts( + floats.values().as_ptr().cast::(), + number_of_points, + ) }); Some(multi_point) @@ -149,7 +155,7 @@ impl GeometryCollection for MultiPointCollection { unsafe { slice::from_raw_parts( - floats.raw_values().cast::(), + floats.values().as_ptr().cast::(), number_of_coordinates, ) } @@ -166,7 +172,7 @@ impl GeometryCollection for MultiPointCollection { let data = geometries.data(); let buffer = &data.buffers()[0]; - unsafe { slice::from_raw_parts(buffer.raw_data().cast::(), geometries.len() + 1) } + unsafe { slice::from_raw_parts(buffer.as_ptr().cast::(), geometries.len() + 1) } } } diff --git a/datatypes/src/collections/multi_polygon_collection.rs b/datatypes/src/collections/multi_polygon_collection.rs index d492334f1..6033511e1 100644 --- a/datatypes/src/collections/multi_polygon_collection.rs +++ b/datatypes/src/collections/multi_polygon_collection.rs @@ -35,7 +35,7 @@ impl GeometryCollection for MultiPolygonCollection { unsafe { slice::from_raw_parts( - floats.raw_values().cast::(), + floats.values().as_ptr().cast::(), number_of_coordinates, ) } @@ -52,7 +52,7 @@ impl GeometryCollection for MultiPolygonCollection { let data = geometries.data(); let buffer = &data.buffers()[0]; - unsafe { slice::from_raw_parts(buffer.raw_data().cast::(), geometries.len() + 1) } + unsafe { slice::from_raw_parts(buffer.as_ptr().cast::(), geometries.len() + 1) } } } @@ -71,7 +71,7 @@ impl MultiPolygonCollection { let data = polygons.data(); let buffer = &data.buffers()[0]; - unsafe { slice::from_raw_parts(buffer.raw_data().cast::(), polygons.len() + 1) } + unsafe { slice::from_raw_parts(buffer.as_ptr().cast::(), polygons.len() + 1) } } #[allow(clippy::cast_ptr_alignment)] @@ -91,7 +91,7 @@ impl MultiPolygonCollection { let data = rings.data(); let buffer = &data.buffers()[0]; - unsafe { slice::from_raw_parts(buffer.raw_data().cast::(), rings.len() + 1) } + unsafe { slice::from_raw_parts(buffer.as_ptr().cast::(), rings.len() + 1) } } } @@ -161,7 +161,7 @@ impl<'l> Iterator for MultiPolygonIterator<'l> { ring_refs.push(unsafe { #[allow(clippy::cast_ptr_alignment)] slice::from_raw_parts( - float_array.raw_values().cast::(), + float_array.values().as_ptr().cast::(), number_of_coordinates, ) }); @@ -225,7 +225,7 @@ impl<'l> GeometryRandomAccess<'l> for MultiPolygonCollection { ring_refs.push(unsafe { #[allow(clippy::cast_ptr_alignment)] slice::from_raw_parts( - float_array.raw_values().cast::(), + float_array.values().as_ptr().cast::(), number_of_coordinates, ) }); diff --git a/datatypes/src/plots/histogram.rs b/datatypes/src/plots/histogram.rs index e37eb4cf4..5c03ffafb 100644 --- a/datatypes/src/plots/histogram.rs +++ b/datatypes/src/plots/histogram.rs @@ -348,7 +348,7 @@ impl HistogramBuilder { #[cfg(test)] mod tests { - use arrow::array::{Array, Float64Builder, Int64Builder, PrimitiveArrayOps, UInt8Builder}; + use arrow::array::{Array, Float64Builder, Int64Builder, UInt8Builder}; use crate::primitives::{CategoricalDataRef, DecimalDataRef, NumberDataRef}; diff --git a/datatypes/src/primitives/coordinate.rs b/datatypes/src/primitives/coordinate.rs index f9dc6dace..ca469552a 100644 --- a/datatypes/src/primitives/coordinate.rs +++ b/datatypes/src/primitives/coordinate.rs @@ -1,6 +1,6 @@ use crate::util::arrow::ArrowTyped; use arrow::array::{ArrayBuilder, BooleanArray, Float64Builder}; -use arrow::datatypes::DataType; +use arrow::datatypes::{DataType, Field}; use arrow::error::ArrowError; use ocl::OclPrm; #[cfg(feature = "postgres")] @@ -138,7 +138,9 @@ impl ArrowTyped for Coordinate2D { type ArrowBuilder = arrow::array::FixedSizeListBuilder; fn arrow_data_type() -> DataType { - arrow::datatypes::DataType::FixedSizeList(Box::new(arrow::datatypes::DataType::Float64), 2) + let nullable = true; // TODO: should actually be false, but arrow's builders set it to `true` currently + + DataType::FixedSizeList(Box::new(Field::new("item", DataType::Float64, nullable)), 2) } fn builder_byte_size(builder: &mut Self::ArrowBuilder) -> usize { diff --git a/datatypes/src/primitives/feature_data.rs b/datatypes/src/primitives/feature_data.rs index b63883f3c..939a34156 100644 --- a/datatypes/src/primitives/feature_data.rs +++ b/datatypes/src/primitives/feature_data.rs @@ -135,7 +135,7 @@ where #[derive(Clone, Debug)] pub struct NumberDataRef<'f> { - buffer: arrow::buffer::Buffer, + buffer: &'f [f64], valid_bitmap: &'f Option, } @@ -173,7 +173,7 @@ impl<'f> DataRef<'f, f64> for NumberDataRef<'f> { impl AsRef<[f64]> for NumberDataRef<'_> { fn as_ref(&self) -> &[f64] { - unsafe { self.buffer.typed_data() } + self.buffer } } @@ -184,10 +184,7 @@ impl<'f> From> for FeatureDataRef<'f> { } impl<'f> NumberDataRef<'f> { - pub fn new( - buffer: arrow::buffer::Buffer, - null_bitmap: &'f Option, - ) -> Self { + pub fn new(buffer: &'f [f64], null_bitmap: &'f Option) -> Self { Self { buffer, valid_bitmap: null_bitmap, @@ -197,15 +194,12 @@ impl<'f> NumberDataRef<'f> { #[derive(Clone, Debug)] pub struct DecimalDataRef<'f> { - buffer: arrow::buffer::Buffer, + buffer: &'f [i64], valid_bitmap: &'f Option, } impl<'f> DecimalDataRef<'f> { - pub fn new( - buffer: arrow::buffer::Buffer, - null_bitmap: &'f Option, - ) -> Self { + pub fn new(buffer: &'f [i64], null_bitmap: &'f Option) -> Self { Self { buffer, valid_bitmap: null_bitmap, @@ -247,7 +241,7 @@ impl<'f> DataRef<'f, i64> for DecimalDataRef<'f> { impl AsRef<[i64]> for DecimalDataRef<'_> { fn as_ref(&self) -> &[i64] { - unsafe { self.buffer.typed_data() } + self.buffer } } @@ -267,7 +261,7 @@ fn null_bitmap_to_bools(null_bitmap: &Option, len: usize) -> Vec { #[derive(Clone, Debug)] pub struct CategoricalDataRef<'f> { - buffer: arrow::buffer::Buffer, + buffer: &'f [u8], valid_bitmap: &'f Option, } @@ -305,7 +299,7 @@ impl<'f> DataRef<'f, u8> for CategoricalDataRef<'f> { impl AsRef<[u8]> for CategoricalDataRef<'_> { fn as_ref(&self) -> &[u8] { - self.buffer.data() + self.buffer } } @@ -316,10 +310,7 @@ impl<'f> From> for FeatureDataRef<'f> { } impl<'f> CategoricalDataRef<'f> { - pub fn new( - buffer: arrow::buffer::Buffer, - null_bitmap: &'f Option, - ) -> Self { + pub fn new(buffer: &'f [u8], null_bitmap: &'f Option) -> Self { Self { buffer, valid_bitmap: null_bitmap, @@ -370,7 +361,7 @@ pub struct TextDataRef<'f> { impl<'f> AsRef<[u8]> for TextDataRef<'f> { fn as_ref(&self) -> &[u8] { - self.data_buffer.data() + self.data_buffer.as_slice() } } @@ -389,7 +380,7 @@ impl<'r> DataRef<'r, u8> for TextDataRef<'r> { let text = unsafe { byte_ptr_to_str( - self.data_buffer.slice(start as usize).raw_data(), + self.data_buffer.slice(start as usize).as_ptr(), (end - start) as usize, ) }; @@ -501,7 +492,7 @@ impl<'r> TextDataRef<'r> { let text = unsafe { byte_ptr_to_str( - self.data_buffer.slice(start as usize).raw_data(), + self.data_buffer.slice(start as usize).as_ptr(), (end - start) as usize, ) }; diff --git a/datatypes/src/primitives/multi_line_string.rs b/datatypes/src/primitives/multi_line_string.rs index 123f07d55..fbd3d1ea1 100644 --- a/datatypes/src/primitives/multi_line_string.rs +++ b/datatypes/src/primitives/multi_line_string.rs @@ -1,6 +1,6 @@ use std::convert::TryFrom; -use arrow::array::{ArrayBuilder, BooleanArray, PrimitiveArrayOps}; +use arrow::array::{ArrayBuilder, BooleanArray}; use arrow::error::ArrowError; use geo::algorithm::intersects::Intersects; use serde::{Deserialize, Serialize}; @@ -8,7 +8,9 @@ use snafu::ensure; use crate::collections::VectorDataType; use crate::error::Error; -use crate::primitives::{error, BoundingBox2D, GeometryRef, PrimitivesError, TypedGeometry}; +use crate::primitives::{ + error, BoundingBox2D, GeometryRef, MultiPoint, PrimitivesError, TypedGeometry, +}; use crate::primitives::{Coordinate2D, Geometry}; use crate::util::arrow::{downcast_array, ArrowTyped}; use crate::util::Result; @@ -106,9 +108,7 @@ impl ArrowTyped for MultiLineString { >; fn arrow_data_type() -> arrow::datatypes::DataType { - arrow::datatypes::DataType::List( - arrow::datatypes::DataType::List(Coordinate2D::arrow_data_type().into()).into(), - ) + MultiPoint::arrow_list_data_type() } fn builder_byte_size(builder: &mut Self::ArrowBuilder) -> usize { @@ -154,9 +154,7 @@ impl ArrowTyped for MultiLineString { let floats_ref = coordinates.value(coordinate_index); let floats: &Float64Array = downcast_array(&floats_ref); - coordinate_builder - .values() - .append_slice(floats.value_slice(0, 2))?; + coordinate_builder.values().append_slice(floats.values())?; coordinate_builder.append(true)?; } @@ -199,9 +197,7 @@ impl ArrowTyped for MultiLineString { let floats_ref = coordinates.value(coordinate_index); let floats: &Float64Array = downcast_array(&floats_ref); - coordinate_builder - .values() - .append_slice(floats.value_slice(0, 2))?; + coordinate_builder.values().append_slice(floats.values())?; coordinate_builder.append(true)?; } diff --git a/datatypes/src/primitives/multi_point.rs b/datatypes/src/primitives/multi_point.rs index 4e0787968..d3f2fe978 100644 --- a/datatypes/src/primitives/multi_point.rs +++ b/datatypes/src/primitives/multi_point.rs @@ -1,6 +1,6 @@ use std::convert::{TryFrom, TryInto}; -use arrow::array::{ArrayBuilder, BooleanArray, PrimitiveArrayOps}; +use arrow::array::{ArrayBuilder, BooleanArray}; use arrow::error::ArrowError; use serde::{Deserialize, Serialize}; use snafu::ensure; @@ -116,7 +116,7 @@ impl ArrowTyped for MultiPoint { type ArrowBuilder = arrow::array::ListBuilder<::ArrowBuilder>; fn arrow_data_type() -> arrow::datatypes::DataType { - arrow::datatypes::DataType::List(Box::new(Coordinate2D::arrow_data_type())) + Coordinate2D::arrow_list_data_type() } fn builder_byte_size(builder: &mut Self::ArrowBuilder) -> usize { @@ -151,7 +151,7 @@ impl ArrowTyped for MultiPoint { let floats: &Float64Array = downcast_array(&floats_ref); let new_floats = new_points.values(); - new_floats.append_slice(floats.value_slice(0, 2))?; + new_floats.append_slice(floats.values())?; new_points.append(true)?; } @@ -184,7 +184,7 @@ impl ArrowTyped for MultiPoint { let old_floats: &Float64Array = downcast_array(&old_floats_array); let float_builder = coordinate_builder.values(); - float_builder.append_slice(old_floats.value_slice(0, 2))?; + float_builder.append_slice(old_floats.values())?; coordinate_builder.append(true)?; } diff --git a/datatypes/src/primitives/multi_polygon.rs b/datatypes/src/primitives/multi_polygon.rs index 4d963c734..d9bccedf4 100644 --- a/datatypes/src/primitives/multi_polygon.rs +++ b/datatypes/src/primitives/multi_polygon.rs @@ -1,6 +1,6 @@ use std::convert::TryFrom; -use arrow::array::{ArrayBuilder, BooleanArray, PrimitiveArrayOps}; +use arrow::array::{ArrayBuilder, BooleanArray}; use arrow::error::ArrowError; use geo::intersects::Intersects; use serde::{Deserialize, Serialize}; @@ -8,10 +8,13 @@ use snafu::ensure; use crate::collections::VectorDataType; use crate::error::Error; -use crate::primitives::{error, BoundingBox2D, GeometryRef, PrimitivesError, TypedGeometry}; +use crate::primitives::{ + error, BoundingBox2D, GeometryRef, MultiLineString, PrimitivesError, TypedGeometry, +}; use crate::primitives::{Coordinate2D, Geometry}; use crate::util::arrow::{downcast_array, ArrowTyped}; use crate::util::Result; +use arrow::datatypes::DataType; /// A trait that allows a common access to polygons of `MultiPolygon`s and its references pub trait MultiPolygonAccess @@ -143,13 +146,8 @@ impl ArrowTyped for MultiPolygon { >, >; - fn arrow_data_type() -> arrow::datatypes::DataType { - arrow::datatypes::DataType::List( - arrow::datatypes::DataType::List( - arrow::datatypes::DataType::List(Coordinate2D::arrow_data_type().into()).into(), - ) - .into(), - ) + fn arrow_data_type() -> DataType { + MultiLineString::arrow_list_data_type() } fn builder_byte_size(builder: &mut Self::ArrowBuilder) -> usize { @@ -208,9 +206,7 @@ impl ArrowTyped for MultiPolygon { let floats_ref = coordinates.value(coordinate_index); let floats: &Float64Array = downcast_array(&floats_ref); - coordinate_builder - .values() - .append_slice(floats.value_slice(0, 2))?; + coordinate_builder.values().append_slice(floats.values())?; coordinate_builder.append(true)?; } @@ -262,9 +258,7 @@ impl ArrowTyped for MultiPolygon { let floats_ref = coordinates.value(coordinate_index); let floats: &Float64Array = downcast_array(&floats_ref); - coordinate_builder - .values() - .append_slice(floats.value_slice(0, 2))?; + coordinate_builder.values().append_slice(floats.values())?; coordinate_builder.append(true)?; } diff --git a/datatypes/src/primitives/no_geometry.rs b/datatypes/src/primitives/no_geometry.rs index aeefe2930..db72d338a 100644 --- a/datatypes/src/primitives/no_geometry.rs +++ b/datatypes/src/primitives/no_geometry.rs @@ -1,9 +1,7 @@ use std::any::Any; use std::convert::TryFrom; -use arrow::array::{ - Array, ArrayBuilder, ArrayDataRef, ArrayEqual, ArrayRef, BooleanArray, JsonEqual, -}; +use arrow::array::{Array, ArrayBuilder, ArrayDataRef, ArrayRef, BooleanArray, JsonEqual}; use arrow::datatypes::DataType; use arrow::error::ArrowError; use serde::{Deserialize, Serialize}; @@ -121,14 +119,6 @@ impl ArrayBuilder for NoArrowArray { unreachable!("There is no implementation since there is no geometry") } - fn append_data(&mut self, _data: &[ArrayDataRef]) -> Result<(), ArrowError> { - unreachable!("There is no implementation since there is no geometry") - } - - fn data_type(&self) -> DataType { - unreachable!("There is no implementation since there is no geometry") - } - fn finish(&mut self) -> ArrayRef { unreachable!("There is no implementation since there is no geometry") } @@ -151,19 +141,3 @@ impl JsonEqual for NoArrowArray { unreachable!("There is no implementation since there is no geometry") } } - -impl ArrayEqual for NoArrowArray { - fn equals(&self, _other: &dyn Array) -> bool { - unreachable!("There is no implementation since there is no geometry") - } - - fn range_equals( - &self, - _other: &dyn Array, - _start_idx: usize, - _end_idx: usize, - _other_start_idx: usize, - ) -> bool { - unreachable!("There is no implementation since there is no geometry") - } -} diff --git a/datatypes/src/primitives/time_interval.rs b/datatypes/src/primitives/time_interval.rs index f4d06dbee..040db81b9 100755 --- a/datatypes/src/primitives/time_interval.rs +++ b/datatypes/src/primitives/time_interval.rs @@ -2,7 +2,8 @@ use crate::error; use crate::primitives::TimeInstance; use crate::util::arrow::{downcast_array, ArrowTyped}; use crate::util::Result; -use arrow::array::{Array, ArrayBuilder, BooleanArray, PrimitiveArrayOps}; +use arrow::array::{Array, ArrayBuilder, BooleanArray}; +use arrow::datatypes::{DataType, Field}; use arrow::error::ArrowError; #[cfg(feature = "postgres")] use postgres_types::{FromSql, ToSql}; @@ -344,12 +345,11 @@ impl ArrowTyped for TimeInterval { fn arrow_data_type() -> arrow::datatypes::DataType { // TODO: use date if dates out-of-range is fixed for us - // arrow::datatypes::DataType::FixedSizeList( - // arrow::datatypes::DataType::Date64(arrow::datatypes::DateUnit::Millisecond).into(), - // 2, - // ) + // DataType::FixedSizeList(Box::new(Field::new("item", DataType::Date64(arrow::datatypes::DateUnit::Millisecond), nullable)), 2) - arrow::datatypes::DataType::FixedSizeList(arrow::datatypes::DataType::Int64.into(), 2) + let nullable = true; // TODO: should actually be false, but arrow's builders set it to `true` currently + + DataType::FixedSizeList(Box::new(Field::new("item", DataType::Int64, nullable)), 2) } fn builder_byte_size(builder: &mut Self::ArrowBuilder) -> usize { @@ -380,8 +380,8 @@ impl ArrowTyped for TimeInterval { let ints_a: &Int64Array = downcast_array(&ints_a_ref); let ints_b: &Int64Array = downcast_array(&ints_b_ref); - int_builder.append_slice(ints_a.value_slice(0, ints_a.len()))?; - int_builder.append_slice(ints_b.value_slice(0, ints_b.len()))?; + int_builder.append_slice(ints_a.values())?; + int_builder.append_slice(ints_b.values())?; } for _ in 0..new_length { @@ -410,7 +410,7 @@ impl ArrowTyped for TimeInterval { let old_timestamps: &Int64Array = downcast_array(&old_timestamps_ref); let date_builder = new_time_intervals.values(); - date_builder.append_slice(old_timestamps.value_slice(0, 2))?; + date_builder.append_slice(old_timestamps.values())?; new_time_intervals.append(true)?; } diff --git a/datatypes/src/util/arrow.rs b/datatypes/src/util/arrow.rs index 07493b390..c272f1ac4 100644 --- a/datatypes/src/util/arrow.rs +++ b/datatypes/src/util/arrow.rs @@ -1,7 +1,7 @@ use std::any::Any; use arrow::array::{Array, ArrayBuilder, ArrayRef, BooleanArray}; -use arrow::datatypes::DataType; +use arrow::datatypes::{DataType, Field}; use arrow::error::ArrowError; /// Helper function to downcast an arrow array @@ -33,6 +33,15 @@ pub trait ArrowTyped { /// Return the specific arrow data type fn arrow_data_type() -> DataType; + fn arrow_list_data_type() -> DataType { + let nullable = true; // TODO: should actually be false, but arrow's builders set it to `true` currently + DataType::List(Box::new(Field::new( + "item", + Self::arrow_data_type(), + nullable, + ))) + } + /// Computes the byte size of the builder fn builder_byte_size(builder: &mut Self::ArrowBuilder) -> usize; diff --git a/datatypes/tests/example-arrow.rs b/datatypes/tests/example-arrow.rs index d4f23f3f7..4108d8b4d 100755 --- a/datatypes/tests/example-arrow.rs +++ b/datatypes/tests/example-arrow.rs @@ -1,12 +1,11 @@ use arrow::array::{ Array, ArrayData, Date64Array, Date64Builder, FixedSizeBinaryBuilder, FixedSizeListArray, FixedSizeListBuilder, Float64Array, Float64Builder, Int32Array, Int32Builder, ListArray, - ListBuilder, PrimitiveArrayOps, StringArray, StringBuilder, StructBuilder, UInt64Array, - UInt64Builder, + ListBuilder, StringArray, StringBuilder, StructBuilder, UInt64Array, UInt64Builder, }; use arrow::buffer::{Buffer, MutableBuffer}; use arrow::compute::kernels::filter::filter; -use arrow::datatypes::{DataType, DateUnit, Field, Schema}; +use arrow::datatypes::{DataType, DateUnit, Field}; use geoengine_datatypes::primitives::{Coordinate2D, TimeInterval}; use ocl::ProQue; use std::{mem, slice}; @@ -47,17 +46,12 @@ fn null_values() { assert_eq!(primitive_array.len(), 5); assert_eq!(primitive_array.null_count(), 1); - let buffer = primitive_array.values(); + let data = primitive_array.values(); - let underlying_data = buffer.data(); - assert_eq!(underlying_data.len(), 5 * 4); + assert_eq!(data.len(), 5); - let casted_data = unsafe { buffer.typed_data::() }; - - assert_eq!(casted_data.len(), 5); - - assert_eq!(&casted_data[0..1], &[1]); - assert_eq!(&casted_data[2..5], &[3, 4, 5]); + assert_eq!(&data[0..1], &[1]); + assert_eq!(&data[2..5], &[3, 4, 5]); } #[test] @@ -78,7 +72,7 @@ fn null_bytes() { assert_eq!(null_bitmap.len(), 1); assert_eq!( - null_bitmap.clone().into_buffer().data(), // must clone bitmap because there is no way to get a reference to the data + null_bitmap.clone().into_buffer().as_slice(), // must clone bitmap because there is no way to get a reference to the data &[0b0000_1001] // right most bit is first element, 1 = valid value, 0 = null or unset ); } @@ -103,12 +97,9 @@ fn offset() { assert_eq!(subarray.len(), 2); assert_eq!(subarray.offset(), 2); - assert_eq!( - unsafe { typed_subarray.values().typed_data::() }.len(), - 5 - ); // does NOT point to sub-slice + assert_eq!(typed_subarray.values().len(), 2); - assert_eq!(array.value_slice(2, 2), &[20., 9.4]); + assert_eq!(typed_subarray.values(), &[20., 9.4]); } #[test] @@ -176,7 +167,7 @@ fn strings2() { assert_eq!(array.value(3), "other"); assert_eq!(array.value(4), "side"); - assert_eq!(array.value_data().data(), b"hellofromtheotherside"); + assert_eq!(array.value_data().as_slice(), b"hellofromtheotherside"); assert_eq!( unsafe { array.value_offsets().typed_data::() }, &[0, 5, 9, 12, 17, 21] @@ -211,7 +202,7 @@ fn list() { .as_any() .downcast_ref::() .unwrap() - .value_slice(0, 5), + .values(), &[0, 1, 2, 3, 4], ); assert_eq!( @@ -250,7 +241,7 @@ fn fixed_size_list() { .as_any() .downcast_ref::() .unwrap() - .value_slice(0, array.len() * array.value_length() as usize), + .values(), &[0, 1, 2, 3, 4, 5], ); } @@ -298,7 +289,7 @@ fn binary() { assert_eq!( unsafe { slice::from_raw_parts( - array.value_data().raw_data() as *const TimeInterval, + array.value_data().as_ptr() as *const TimeInterval, array.len(), ) }, @@ -338,7 +329,7 @@ fn ocl() { let ocl_buffer = pro_que .buffer_builder() - .copy_host_slice(array.value_slice(0, array.len())) + .copy_host_slice(array.values()) .build() .unwrap(); @@ -358,18 +349,18 @@ fn ocl() { let result = { let buffer = MutableBuffer::new(ocl_buffer.len() * mem::size_of::()); let buffer_raw: &mut [i32] = - unsafe { slice::from_raw_parts_mut(buffer.raw_data() as *mut i32, ocl_buffer.len()) }; + unsafe { slice::from_raw_parts_mut(buffer.as_ptr() as *mut i32, ocl_buffer.len()) }; ocl_buffer.read(buffer_raw).enq().unwrap(); let data = ArrayData::builder(DataType::Int32) .len(ocl_buffer.len()) - .add_buffer(buffer.freeze()) + .add_buffer(buffer.into()) .build(); Int32Array::from(data) }; - assert_eq!(result.value_slice(0, result.len()), &[11, 12, 13, 14, 15]); + assert_eq!(result.values(), &[11, 12, 13, 14, 15]); } #[test] @@ -386,20 +377,20 @@ fn serialize() { assert_eq!(array.len(), 5); // no serialization of arrays by now - let json = serde_json::to_string(array.value_slice(0, array.len())).unwrap(); + let json = serde_json::to_string(array.values()).unwrap(); assert_eq!(json, "[1,2,3,4,5]"); } #[test] fn table() { - let schema = Schema::new(vec![ + let schema = vec![ Field::new("feature_start", DataType::UInt64, false), Field::new("time_start", DataType::Date64(DateUnit::Millisecond), false), - ]); + ]; let array = { - let mut builder = StructBuilder::from_schema(schema, 5); + let mut builder = StructBuilder::from_fields(schema, 5); for &(feature_start, time) in &[(0_u64, 0_i64), (1, 10), (2, 20), (3, 30), (4, 40)] { builder @@ -426,7 +417,7 @@ fn table() { .as_any() .downcast_ref::() .unwrap() - .value_slice(0, array.len()), + .values(), &[0, 1, 2, 3, 4] ); assert_eq!( @@ -436,7 +427,7 @@ fn table() { .as_any() .downcast_ref::() .unwrap() - .value_slice(0, array.len()), + .values(), &[0, 10, 20, 30, 40] ); } @@ -524,27 +515,32 @@ fn multipoints() { use arrow::datatypes::ToByteSlice; let array = { - let data = ArrayData::builder(DataType::List( - DataType::FixedSizeList(DataType::Float64.into(), 2).into(), - )) + let data = ArrayData::builder(DataType::List(Box::new(Field::new( + "", + DataType::FixedSizeList(Box::new(Field::new("", DataType::Float64, false)), 2), + false, + )))) .len(2) // number of multipoints .add_buffer(Buffer::from(&[0_i32, 2, 5].to_byte_slice())) .add_child_data( - ArrayData::builder(DataType::FixedSizeList(DataType::Float64.into(), 2)) - .len(5) // number of coordinates - .add_child_data( - ArrayData::builder(DataType::Float64) - .len(10) // number of floats - .add_buffer(Buffer::from( - &[ - 1_f64, 2., 11., 12., 21., 22., 31., 32., 41., 42., 51., 52., 61., - 62., 71., 72., 81., 82., 91., 92., - ] - .to_byte_slice(), - )) - .build(), - ) - .build(), + ArrayData::builder(DataType::FixedSizeList( + Box::new(Field::new("", DataType::Float64, false)), + 2, + )) + .len(5) // number of coordinates + .add_child_data( + ArrayData::builder(DataType::Float64) + .len(10) // number of floats + .add_buffer(Buffer::from( + &[ + 1_f64, 2., 11., 12., 21., 22., 31., 32., 41., 42., 51., 52., 61., 62., + 71., 72., 81., 82., 91., 92., + ] + .to_byte_slice(), + )) + .build(), + ) + .build(), ) .build(); @@ -565,7 +561,7 @@ fn multipoints() { .as_any() .downcast_ref::() .unwrap() - .value_slice(0, 10); + .values(); assert_eq!(floats.len(), 10); let coordinates: &[Coordinate2D] = unsafe { slice::from_raw_parts(floats.as_ptr() as *const Coordinate2D, floats.len()) }; @@ -624,7 +620,7 @@ fn multipoint_builder() { let coordinates_ref = first_multi_point.values(); let coordinates: &Float64Array = coordinates_ref.as_any().downcast_ref().unwrap(); - assert_eq!(coordinates.value_slice(0, 2 * 2), &[0.0, 0.1, 1.0, 1.1]); + assert_eq!(&coordinates.values()[0..2 * 2], &[0.0, 0.1, 1.0, 1.1]); let second_multi_point_ref = multi_point.value(1); let second_multi_point: &arrow::array::FixedSizeListArray = @@ -702,3 +698,35 @@ fn multipoint_builder_bytes() { &[(2.0, 2.1).into(), (3.0, 3.1).into(), (4.0, 4.1).into()] ); } + +#[test] +#[allow(clippy::eq_op)] +fn float_equality() { + let mut floats = Float64Builder::new(3); + floats.append_value(4.0).unwrap(); + floats.append_null().unwrap(); + floats.append_value(f64::NAN).unwrap(); + + let floats = floats.finish(); + + assert_eq!(floats, floats); + + let mut floats2 = Float64Builder::new(3); + floats2.append_value(4.0).unwrap(); + floats2.append_null().unwrap(); + floats2.append_value(f64::NAN).unwrap(); + + let floats2 = floats2.finish(); + + assert_eq!(floats, floats2); + + let mut floats3 = Float64Builder::new(3); + floats3.append_value(f64::NAN).unwrap(); + floats3.append_null().unwrap(); + floats3.append_value(4.0).unwrap(); + + let floats3 = floats3.finish(); + + assert_ne!(floats, floats3); + assert_ne!(floats2, floats3); +} diff --git a/operators/Cargo.toml b/operators/Cargo.toml index 65b3b8342..b4b2072aa 100644 --- a/operators/Cargo.toml +++ b/operators/Cargo.toml @@ -9,7 +9,8 @@ authors = [ edition = "2018" [dependencies] -arrow = { version = "2.0", features = ["simd"] } +# arrow = { version = "3.0", features = ["simd"] } TODO: activate when version is patched +arrow = { version = "3.0" } chrono = "0.4" crossbeam = "0.8" csv = "1.1" diff --git a/operators/src/mock/mock_feature_collection_source.rs b/operators/src/mock/mock_feature_collection_source.rs index b86ec1824..bd1f1727d 100644 --- a/operators/src/mock/mock_feature_collection_source.rs +++ b/operators/src/mock/mock_feature_collection_source.rs @@ -153,59 +153,62 @@ mod tests { let serialized = serde_json::to_string(&source).unwrap(); let collection_bytes = [ - 65, 82, 82, 79, 87, 49, 0, 0, 148, 1, 0, 0, 16, 0, 0, 0, 0, 0, 10, 0, 14, 0, 12, 0, 11, + 65, 82, 82, 79, 87, 49, 0, 0, 180, 1, 0, 0, 16, 0, 0, 0, 0, 0, 10, 0, 14, 0, 12, 0, 11, 0, 4, 0, 10, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 1, 3, 0, 10, 0, 12, 0, 0, 0, 8, 0, 4, 0, - 10, 0, 0, 0, 8, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 180, 0, 0, 0, 72, 0, 0, 0, - 20, 0, 0, 0, 16, 0, 20, 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, 16, 0, - 0, 0, 24, 0, 0, 0, 0, 0, 1, 2, 20, 0, 0, 0, 176, 255, 255, 255, 64, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 6, 0, 0, 0, 102, 111, 111, 98, 97, 114, 0, 0, 168, 255, 255, 255, 24, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 16, 60, 0, 0, 0, 1, 0, 0, 0, 12, 0, 0, 0, 102, 255, 255, - 255, 2, 0, 0, 0, 152, 255, 255, 255, 0, 0, 0, 2, 16, 0, 0, 0, 24, 0, 0, 0, 8, 0, 12, 0, - 4, 0, 11, 0, 8, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 95, 95, 116, - 105, 109, 101, 0, 0, 16, 0, 20, 0, 16, 0, 0, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, - 28, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 12, 128, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 4, 0, 4, - 0, 4, 0, 0, 0, 16, 0, 16, 0, 0, 0, 0, 0, 7, 0, 8, 0, 0, 0, 12, 0, 16, 0, 0, 0, 0, 0, 0, - 16, 24, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 36, 0, 0, 0, 0, 0, 6, 0, 8, 0, 4, 0, 6, 0, 0, - 0, 2, 0, 0, 0, 16, 0, 18, 0, 0, 0, 0, 0, 7, 0, 8, 0, 0, 0, 12, 0, 16, 0, 0, 0, 0, 0, 0, - 3, 16, 0, 0, 0, 20, 0, 0, 0, 0, 0, 6, 0, 8, 0, 6, 0, 6, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, - 0, 10, 0, 0, 0, 95, 95, 103, 101, 111, 109, 101, 116, 114, 121, 0, 0, 0, 0, 0, 0, 92, - 1, 0, 0, 16, 0, 0, 0, 12, 0, 26, 0, 24, 0, 23, 0, 4, 0, 8, 0, 12, 0, 0, 0, 32, 0, 0, 0, - 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 10, 0, 24, 0, 12, 0, 8, 0, 4, - 0, 10, 0, 0, 0, 124, 0, 0, 0, 16, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, - 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, - 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, - 0, 8, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, - 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, - 0, 96, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, - 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, - 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 2, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 154, 153, 153, 153, 153, 153, 185, 63, 0, 0, 0, 0, 0, 0, 240, 63, 154, 153, - 153, 153, 153, 153, 241, 63, 0, 0, 0, 0, 0, 0, 0, 64, 205, 204, 204, 204, 204, 204, 8, - 64, 7, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 12, 0, 18, 0, 16, 0, 12, 0, - 8, 0, 4, 0, 12, 0, 0, 0, 128, 1, 0, 0, 156, 1, 0, 0, 16, 0, 0, 0, 3, 0, 10, 0, 12, 0, - 0, 0, 8, 0, 4, 0, 10, 0, 0, 0, 8, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 180, 0, - 0, 0, 72, 0, 0, 0, 20, 0, 0, 0, 16, 0, 20, 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, 0, 8, 0, - 16, 0, 0, 0, 16, 0, 0, 0, 24, 0, 0, 0, 0, 0, 1, 2, 20, 0, 0, 0, 176, 255, 255, 255, 64, - 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 102, 111, 111, 98, 97, 114, 0, 0, 168, - 255, 255, 255, 24, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 16, 60, 0, 0, 0, 1, 0, 0, 0, 12, 0, - 0, 0, 102, 255, 255, 255, 2, 0, 0, 0, 152, 255, 255, 255, 0, 0, 0, 2, 16, 0, 0, 0, 24, - 0, 0, 0, 8, 0, 12, 0, 4, 0, 11, 0, 8, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, - 0, 0, 0, 95, 95, 116, 105, 109, 101, 0, 0, 16, 0, 20, 0, 16, 0, 0, 0, 15, 0, 4, 0, 0, - 0, 8, 0, 16, 0, 0, 0, 28, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 12, 128, 0, 0, 0, 1, 0, 0, 0, - 28, 0, 0, 0, 4, 0, 4, 0, 4, 0, 0, 0, 16, 0, 16, 0, 0, 0, 0, 0, 7, 0, 8, 0, 0, 0, 12, 0, - 16, 0, 0, 0, 0, 0, 0, 16, 24, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 36, 0, 0, 0, 0, 0, 6, 0, - 8, 0, 4, 0, 6, 0, 0, 0, 2, 0, 0, 0, 16, 0, 18, 0, 0, 0, 0, 0, 7, 0, 8, 0, 0, 0, 12, 0, - 16, 0, 0, 0, 0, 0, 0, 3, 16, 0, 0, 0, 20, 0, 0, 0, 0, 0, 6, 0, 8, 0, 6, 0, 6, 0, 0, 0, - 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 0, 0, 95, 95, 103, 101, 111, 109, 101, 116, 114, 121, 0, - 0, 1, 0, 0, 0, 160, 1, 0, 0, 0, 0, 0, 0, 96, 1, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 1, 0, 0, 65, 82, 82, 79, 87, 49, + 10, 0, 0, 0, 8, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 180, 0, 0, 0, 56, 0, 0, 0, + 4, 0, 0, 0, 52, 255, 255, 255, 16, 0, 0, 0, 24, 0, 0, 0, 0, 0, 1, 2, 20, 0, 0, 0, 172, + 255, 255, 255, 64, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 102, 111, 111, 98, 97, + 114, 0, 0, 152, 255, 255, 255, 24, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 16, 76, 0, 0, 0, 1, + 0, 0, 0, 12, 0, 0, 0, 82, 255, 255, 255, 2, 0, 0, 0, 136, 255, 255, 255, 24, 0, 0, 0, + 32, 0, 0, 0, 0, 0, 1, 2, 28, 0, 0, 0, 8, 0, 12, 0, 4, 0, 11, 0, 8, 0, 0, 0, 64, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 105, 116, 101, 109, 0, 0, 0, 0, 6, 0, 0, 0, 95, + 95, 116, 105, 109, 101, 0, 0, 16, 0, 20, 0, 16, 0, 0, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, + 0, 0, 0, 28, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 12, 160, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, + 4, 0, 4, 0, 4, 0, 0, 0, 16, 0, 20, 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, + 0, 32, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 16, 96, 0, 0, 0, 1, 0, 0, 0, 36, 0, 0, 0, 0, 0, + 6, 0, 8, 0, 4, 0, 6, 0, 0, 0, 2, 0, 0, 0, 16, 0, 22, 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, + 0, 8, 0, 16, 0, 0, 0, 24, 0, 0, 0, 28, 0, 0, 0, 0, 0, 1, 3, 24, 0, 0, 0, 0, 0, 6, 0, 8, + 0, 6, 0, 6, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 105, 116, 101, 109, 0, 0, 0, + 0, 4, 0, 0, 0, 105, 116, 101, 109, 0, 0, 0, 0, 10, 0, 0, 0, 95, 95, 103, 101, 111, 109, + 101, 116, 114, 121, 0, 0, 0, 0, 0, 0, 92, 1, 0, 0, 16, 0, 0, 0, 12, 0, 26, 0, 24, 0, + 23, 0, 4, 0, 8, 0, 12, 0, 0, 0, 32, 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 3, 3, 0, 10, 0, 24, 0, 12, 0, 8, 0, 4, 0, 10, 0, 0, 0, 124, 0, 0, 0, 16, 0, 0, 0, + 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, + 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, + 88, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, + 0, 8, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 153, 153, 153, 153, + 153, 185, 63, 0, 0, 0, 0, 0, 0, 240, 63, 154, 153, 153, 153, 153, 153, 241, 63, 0, 0, + 0, 0, 0, 0, 0, 64, 205, 204, 204, 204, 204, 204, 8, 64, 7, 0, 0, 0, 0, 0, 0, 0, 255, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 16, 0, 0, 0, 12, 0, 18, 0, 16, 0, 12, 0, 8, 0, 4, 0, 12, 0, 0, 0, 160, 1, 0, + 0, 188, 1, 0, 0, 16, 0, 0, 0, 3, 0, 10, 0, 12, 0, 0, 0, 8, 0, 4, 0, 10, 0, 0, 0, 8, 0, + 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 180, 0, 0, 0, 56, 0, 0, 0, 4, 0, 0, 0, 52, + 255, 255, 255, 16, 0, 0, 0, 24, 0, 0, 0, 0, 0, 1, 2, 20, 0, 0, 0, 172, 255, 255, 255, + 64, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 102, 111, 111, 98, 97, 114, 0, 0, 152, + 255, 255, 255, 24, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 16, 76, 0, 0, 0, 1, 0, 0, 0, 12, 0, + 0, 0, 82, 255, 255, 255, 2, 0, 0, 0, 136, 255, 255, 255, 24, 0, 0, 0, 32, 0, 0, 0, 0, + 0, 1, 2, 28, 0, 0, 0, 8, 0, 12, 0, 4, 0, 11, 0, 8, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 1, 0, + 0, 0, 0, 4, 0, 0, 0, 105, 116, 101, 109, 0, 0, 0, 0, 6, 0, 0, 0, 95, 95, 116, 105, 109, + 101, 0, 0, 16, 0, 20, 0, 16, 0, 0, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, 28, 0, 0, + 0, 12, 0, 0, 0, 0, 0, 0, 12, 160, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 4, 0, 4, 0, 4, 0, + 0, 0, 16, 0, 20, 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, 32, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 1, 16, 96, 0, 0, 0, 1, 0, 0, 0, 36, 0, 0, 0, 0, 0, 6, 0, 8, 0, 4, 0, + 6, 0, 0, 0, 2, 0, 0, 0, 16, 0, 22, 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, + 0, 24, 0, 0, 0, 28, 0, 0, 0, 0, 0, 1, 3, 24, 0, 0, 0, 0, 0, 6, 0, 8, 0, 6, 0, 6, 0, 0, + 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 105, 116, 101, 109, 0, 0, 0, 0, 4, 0, 0, 0, 105, + 116, 101, 109, 0, 0, 0, 0, 10, 0, 0, 0, 95, 95, 103, 101, 111, 109, 101, 116, 114, 121, + 0, 0, 1, 0, 0, 0, 192, 1, 0, 0, 0, 0, 0, 0, 96, 1, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 1, 0, 0, 65, 82, 82, 79, 87, 49, ] .to_vec(); assert_eq!( diff --git a/operators/src/opencl/cl_program.rs b/operators/src/opencl/cl_program.rs index 38f55fb97..77e707bd2 100644 --- a/operators/src/opencl/cl_program.rs +++ b/operators/src/opencl/cl_program.rs @@ -1177,14 +1177,14 @@ impl<'a> CLProgramRunnable<'a> { let num_bytes = bit_util::ceil(nulls.len(), 8); let mut arrow_buffer = MutableBuffer::new(num_bytes).with_bitset(num_bytes, false); - let null_slice = arrow_buffer.data_mut(); + let null_slice = arrow_buffer.as_slice_mut(); for (i, null) in nulls.iter().enumerate() { if *null != 0 { bit_util::set_bit(null_slice, i); } } - Some(arrow_buffer.freeze()) + Some(arrow_buffer.into()) } else { None }; @@ -1208,14 +1208,14 @@ impl<'a> CLProgramRunnable<'a> { let num_bytes = bit_util::ceil(nulls.len(), 8); let mut arrow_buffer = MutableBuffer::new(num_bytes).with_bitset(num_bytes, false); - let null_slice = arrow_buffer.data_mut(); + let null_slice = arrow_buffer.as_slice_mut(); for (i, null) in nulls.iter().enumerate() { if *null != 0 { bit_util::set_bit(null_slice, i); } } - Some(arrow_buffer.freeze()) + Some(arrow_buffer.into()) } else { None }; @@ -1239,15 +1239,14 @@ impl<'a> CLProgramRunnable<'a> { len: usize, ) -> Result { let mut arrow_buffer = MutableBuffer::new(len * std::mem::size_of::()); - arrow_buffer.resize(len * std::mem::size_of::()).unwrap(); + arrow_buffer.resize(len * std::mem::size_of::()); - let dest = unsafe { - std::slice::from_raw_parts_mut(arrow_buffer.data_mut().as_ptr() as *mut T, len) - }; + let dest = + unsafe { std::slice::from_raw_parts_mut(arrow_buffer.as_mut_ptr().cast::(), len) }; ocl_buffer.read(dest).enq()?; - Ok(arrow_buffer.freeze()) + Ok(arrow_buffer.into()) } } @@ -2460,17 +2459,14 @@ __kernel void nop(__global int* buffer) { assert_eq!(vec, &[0, 1, 2, 3]); let mut arrow_buffer = MutableBuffer::new(len * std::mem::size_of::()); - arrow_buffer - .resize(len * std::mem::size_of::()) - .unwrap(); + arrow_buffer.resize(len * std::mem::size_of::()); - let dest = unsafe { - std::slice::from_raw_parts_mut(arrow_buffer.data_mut().as_ptr() as *mut i32, len) - }; + let dest = + unsafe { std::slice::from_raw_parts_mut(arrow_buffer.as_mut_ptr().cast::(), len) }; ocl_buffer.read(dest).enq().unwrap(); - let arrow_buffer = arrow_buffer.freeze(); + let arrow_buffer = arrow_buffer.into(); let data = ArrayData::builder(DataType::Int32) .len(len) @@ -2479,7 +2475,7 @@ __kernel void nop(__global int* buffer) { let array = Arc::new(PrimitiveArray::::from(data)); - assert_eq!(array.value_slice(0, len), &[0, 1, 2, 3]); + assert_eq!(array.values(), &[0, 1, 2, 3]); } #[test]