Skip to content

ARROW-10261: [Rust] [Breaking] Change List datatype to Box<Field> #8608

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
nevi-me wants to merge 6 commits into from
Closed

ARROW-10261: [Rust] [Breaking] Change List datatype to Box<Field> #8608

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9dc7994
ARROW-10261: [Rust] Change list types to use Field
nevi-me Oct 11, 2020
d53b839
Change Arrow lists to using Field type
nevi-me Nov 7, 2020
8af39e4
Add Datafusion changes to List<Box<Field>>
nevi-me Nov 7, 2020
8ae32fd
fix failing tests
nevi-me Nov 7, 2020
790da1a
address queries, remove commented out code
nevi-me Nov 7, 2020
dd8178c
rebase
nevi-me Nov 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion rust/arrow/examples/builders.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@ fn main() {
let value_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());

// Construct a list array from the above two
let list_data_type = DataType::List(Box::new(DataType::Int32));
let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.len(3)
.add_buffer(value_offsets)
Expand Down
51 changes: 35 additions & 16 deletions rust/arrow/src/array/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2927,7 +2927,8 @@ mod tests {
let value_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());

// Construct a list array from the above two
let list_data_type = DataType::List(Box::new(DataType::Int32));
let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type.clone())
.len(3)
.add_buffer(value_offsets.clone())
Expand Down Expand Up @@ -2996,7 +2997,8 @@ mod tests {
let value_offsets = Buffer::from(&[0i64, 3, 6, 8].to_byte_slice());

// Construct a list array from the above two
let list_data_type = DataType::LargeList(Box::new(DataType::Int32));
let list_data_type =
DataType::LargeList(Box::new(Field::new("item", DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type.clone())
.len(3)
.add_buffer(value_offsets.clone())
Expand Down Expand Up @@ -3147,7 +3149,10 @@ mod tests {
.build();

// Construct a list array from the above two
let list_data_type = DataType::FixedSizeList(Box::new(DataType::Int32), 3);
let list_data_type = DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Int32, false)),
3,
);
let list_data = ArrayData::builder(list_data_type.clone())
.len(3)
.add_child_data(value_data.clone())
Expand Down Expand Up @@ -3213,7 +3218,10 @@ mod tests {
.build();

// Construct a list array from the above two
let list_data_type = DataType::FixedSizeList(Box::new(DataType::Int32), 3);
let list_data_type = DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Int32, false)),
3,
);
let list_data = ArrayData::builder(list_data_type)
.len(3)
.add_child_data(value_data)
Expand Down Expand Up @@ -3244,7 +3252,8 @@ mod tests {
bit_util::set_bit(&mut null_bits, 8);

// Construct a list array from the above two
let list_data_type = DataType::List(Box::new(DataType::Int32));
let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.len(9)
.add_buffer(value_offsets)
Expand Down Expand Up @@ -3308,7 +3317,8 @@ mod tests {
bit_util::set_bit(&mut null_bits, 8);

// Construct a list array from the above two
let list_data_type = DataType::LargeList(Box::new(DataType::Int32));
let list_data_type =
DataType::LargeList(Box::new(Field::new("item", DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.len(9)
.add_buffer(value_offsets)
Expand Down Expand Up @@ -3370,7 +3380,10 @@ mod tests {
bit_util::set_bit(&mut null_bits, 4);

// Construct a fixed size list array from the above two
let list_data_type = DataType::FixedSizeList(Box::new(DataType::Int32), 2);
let list_data_type = DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Int32, false)),
2,
);
let list_data = ArrayData::builder(list_data_type)
.len(5)
.add_child_data(value_data.clone())
Expand Down Expand Up @@ -3418,7 +3431,8 @@ mod tests {
.len(8)
.add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
.build();
let list_data_type = DataType::List(Box::new(DataType::Int32));
let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.len(3)
.add_child_data(value_data)
Expand All @@ -3432,7 +3446,8 @@ mod tests {
)]
fn test_list_array_invalid_child_array_len() {
let value_offsets = Buffer::from(&[0, 2, 5, 7].to_byte_slice());
let list_data_type = DataType::List(Box::new(DataType::Int32));
let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.len(3)
.add_buffer(value_offsets)
Expand All @@ -3450,7 +3465,8 @@ mod tests {

let value_offsets = Buffer::from(&[2, 2, 5, 7].to_byte_slice());

let list_data_type = DataType::List(Box::new(DataType::Int32));
let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.len(3)
.add_buffer(value_offsets)
Expand Down Expand Up @@ -3843,11 +3859,13 @@ mod tests {
.add_child_data(ArrayData::builder(DataType::Boolean).build())
.build();

let array_data =
ArrayData::builder(DataType::FixedSizeList(Box::new(DataType::Binary), 4))
.len(3)
.add_child_data(values_data)
.build();
let array_data = ArrayData::builder(DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Binary, false)),
4,
))
.len(3)
.add_child_data(values_data)
.build();
let list_array = FixedSizeListArray::from(array_data);
FixedSizeBinaryArray::from(list_array);
}
Expand Down Expand Up @@ -4197,7 +4215,8 @@ mod tests {
.add_buffer(Buffer::from(values.to_byte_slice()))
.build();

let list_data_type = DataType::List(Box::new(DataType::Int32));
let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.add_buffer(buf2)
.add_child_data(value_data)
Expand Down
89 changes: 62 additions & 27 deletions rust/arrow/src/array/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -775,7 +775,11 @@ where
///
/// This is used for validating array data types in `append_data`
fn data_type(&self) -> DataType {
DataType::List(Box::new(self.values_builder.data_type()))
DataType::List(Box::new(Field::new(
"item",
self.values_builder.data_type(),
true,
)))
}

/// Returns the builder as a mutable `Any` reference.
Expand Down Expand Up @@ -839,15 +843,19 @@ where

let offset_buffer = self.offsets_builder.finish();
let null_bit_buffer = self.bitmap_builder.finish();
let nulls = bit_util::count_set_bits(null_bit_buffer.data());
self.offsets_builder.append(0).unwrap();
let data =
ArrayData::builder(DataType::List(Box::new(values_data.data_type().clone())))
.len(len)
.null_count(len - bit_util::count_set_bits(null_bit_buffer.data()))
.add_buffer(offset_buffer)
.add_child_data(values_data)
.null_bit_buffer(null_bit_buffer)
.build();
let data = ArrayData::builder(DataType::List(Box::new(Field::new(
"item",
values_data.data_type().clone(),
true, // TODO: find a consistent way of getting this
))))
.len(len)
.null_count(len - nulls)
.add_buffer(offset_buffer)
.add_child_data(values_data)
.null_bit_buffer(null_bit_buffer)
.build();

ListArray::from(data)
}
Expand Down Expand Up @@ -980,7 +988,11 @@ where
///
/// This is used for validating array data types in `append_data`
fn data_type(&self) -> DataType {
DataType::LargeList(Box::new(self.values_builder.data_type()))
DataType::LargeList(Box::new(Field::new(
"item",
self.values_builder.data_type(),
true,
)))
}

/// Returns the builder as a mutable `Any` reference.
Expand Down Expand Up @@ -1044,12 +1056,15 @@ where

let offset_buffer = self.offsets_builder.finish();
let null_bit_buffer = self.bitmap_builder.finish();
let nulls = bit_util::count_set_bits(null_bit_buffer.data());
self.offsets_builder.append(0).unwrap();
let data = ArrayData::builder(DataType::LargeList(Box::new(
let data = ArrayData::builder(DataType::LargeList(Box::new(Field::new(
"item",
values_data.data_type().clone(),
)))
true,
))))
.len(len)
.null_count(len - bit_util::count_set_bits(null_bit_buffer.data()))
.null_count(len - nulls)
.add_buffer(offset_buffer)
.add_child_data(values_data)
.null_bit_buffer(null_bit_buffer)
Expand Down Expand Up @@ -1155,7 +1170,10 @@ where
///
/// This is used for validating array data types in `append_data`
fn data_type(&self) -> DataType {
DataType::FixedSizeList(Box::new(self.values_builder.data_type()), self.list_len)
DataType::FixedSizeList(
Box::new(Field::new("item", self.values_builder.data_type(), true)),
self.list_len,
)
}

/// Returns the builder as a mutable `Any` reference.
Expand Down Expand Up @@ -1230,12 +1248,13 @@ where
}

let null_bit_buffer = self.bitmap_builder.finish();
let nulls = bit_util::count_set_bits(null_bit_buffer.data());
let data = ArrayData::builder(DataType::FixedSizeList(
Box::new(values_data.data_type().clone()),
Box::new(Field::new("item", values_data.data_type().clone(), true)),
self.list_len,
))
.len(len)
.null_count(len - bit_util::count_set_bits(null_bit_buffer.data()))
.null_count(len - nulls)
.add_child_data(values_data)
.null_bit_buffer(null_bit_buffer)
.build();
Expand Down Expand Up @@ -1445,7 +1464,7 @@ fn append_binary_data(
)) as ArrayDataRef;

Arc::new(ArrayData::new(
DataType::List(Box::new(DataType::UInt8)),
DataType::List(Box::new(Field::new("item", DataType::UInt8, true))),
array.len(),
None,
array.null_buffer().cloned(),
Expand Down Expand Up @@ -1497,7 +1516,11 @@ fn append_large_binary_data(
)) as ArrayDataRef;

Arc::new(ArrayData::new(
DataType::LargeList(Box::new(DataType::UInt8)),
DataType::LargeList(Box::new(Field::new(
"item",
DataType::UInt8,
true,
))),
array.len(),
None,
array.null_buffer().cloned(),
Expand Down Expand Up @@ -1595,7 +1618,10 @@ impl ArrayBuilder for FixedSizeBinaryBuilder {
vec![],
)) as ArrayDataRef;
let list_data = Arc::new(ArrayData::new(
DataType::FixedSizeList(Box::new(DataType::UInt8), self.builder.list_len),
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::UInt8, true)),
self.builder.list_len,
),
array.len(),
None,
array.null_buffer().cloned(),
Expand Down Expand Up @@ -3368,11 +3394,14 @@ mod tests {
}

#[test]
#[should_panic(expected = "Data type List(Int64) is not currently supported")]
#[should_panic(
expected = "Data type List(Field { name: \"item\", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false }) is not currently supported"
)]
fn test_struct_array_builder_from_schema_unsupported_type() {
let mut fields = Vec::new();
fields.push(Field::new("f1", DataType::Int16, false));
let list_type = DataType::List(Box::new(DataType::Int64));
let list_type =
DataType::List(Box::new(Field::new("item", DataType::Int64, true)));
fields.push(Field::new("f2", list_type, false));

let _ = StructBuilder::from_fields(fields, 5);
Expand Down Expand Up @@ -3667,7 +3696,7 @@ mod tests {
let list_value_offsets =
Buffer::from(&[0, 3, 5, 11, 13, 13, 15, 15, 17].to_byte_slice());
let expected_list_data = ArrayData::new(
DataType::List(Box::new(DataType::Int64)),
DataType::List(Box::new(Field::new("item", DataType::Int64, true))),
8,
None,
None,
Expand Down Expand Up @@ -3753,7 +3782,7 @@ mod tests {
&[0, 3, 5, 5, 13, 15, 15, 15, 19, 19, 19, 19, 23].to_byte_slice(),
);
let expected_list_data = ArrayData::new(
DataType::List(Box::new(DataType::Int64)),
DataType::List(Box::new(Field::new("item", DataType::Int64, true))),
12,
None,
None,
Expand Down Expand Up @@ -3795,7 +3824,7 @@ mod tests {
]);
let list_value_offsets = Buffer::from(&[0, 2, 3, 6].to_byte_slice());
let list_data = ArrayData::new(
DataType::List(Box::new(DataType::Utf8)),
DataType::List(Box::new(Field::new("item", DataType::Utf8, true))),
3,
None,
None,
Expand Down Expand Up @@ -3830,7 +3859,7 @@ mod tests {
]);
let list_value_offsets = Buffer::from(&[0, 2, 2, 4, 5, 8, 9, 12].to_byte_slice());
let expected_list_data = ArrayData::new(
DataType::List(Box::new(DataType::Utf8)),
DataType::List(Box::new(Field::new("item", DataType::Utf8, true))),
7,
None,
None, // is this correct?
Expand Down Expand Up @@ -3918,7 +3947,10 @@ mod tests {
Some(12),
]);
let expected_list_data = ArrayData::new(
DataType::FixedSizeList(Box::new(DataType::UInt16), 2),
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::UInt16, true)),
2,
),
12,
None,
None,
Expand Down Expand Up @@ -3988,7 +4020,10 @@ mod tests {
None,
]);
let expected_list_data = ArrayData::new(
DataType::FixedSizeList(Box::new(DataType::UInt8), 2),
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::UInt8, true)),
2,
),
12,
None,
None,
Expand Down
Loading