Skip to content

ARROW-10656: [Rust] Remove Field from (most) DataTypes, add nullability #8715

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ch-sc wants to merge 18 commits into from
Closed

ARROW-10656: [Rust] Remove Field from (most) DataTypes, add nullability #8715

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
5d1d018
ARROW-10656 Use DataType comparison without values
ch-sc Nov 19, 2020
6cffff2
ARROW-10656 invert if condition
ch-sc Nov 19, 2020
4324316
ARROW-10656 Address clippy error
ch-sc Nov 19, 2020
e680fc0
ARROW-10656 Remove if else
ch-sc Nov 20, 2020
5febab8
ARROW-10656 Rename cmp_type to eq_type
ch-sc Nov 20, 2020
d527e16
ARROW-10656 Rename cmp_type to eq_type
ch-sc Nov 21, 2020
ccec7df
ARROW-10656 Introduce data type context
ch-sc Nov 23, 2020
b6cb3dd
Merge branch 'master' of github.com:apache/arrow into ARROW-10656-Rec…
ch-sc Nov 23, 2020
3ab2126
ARROW-10656 Fix test after merging
ch-sc Nov 23, 2020
d62749f
ARROW-10656 keep prior conversion interface
ch-sc Nov 24, 2020
4cdd169
Rename DataTypeContext to NullableDataType
ch-sc Nov 24, 2020
8264a66
More renaming
ch-sc Nov 24, 2020
28829a2
Update rust/arrow/src/datatypes.rs
ch-sc Nov 30, 2020
ae41a95
Merge branch 'ARROW-10656-RecordBatch-requires-exact-data-type-match'…
ch-sc Nov 30, 2020
400eeef
Manual serialization and deserialization of fields
ch-sc Nov 30, 2020
6228f04
fix clippy errors
ch-sc Dec 3, 2020
8de4eb3
Merge branch 'master' of github.com:apache/arrow into ARROW-10656-Rec…
ch-sc Dec 3, 2020
2cccd08
Fixes after merging
ch-sc Dec 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions rust/arrow/examples/builders.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@ use arrow::array::{
StringArray, StructArray,
};
use arrow::buffer::Buffer;
use arrow::datatypes::{DataType, Date64Type, Field, Time64NanosecondType, ToByteSlice};
use arrow::datatypes::{
DataType, Date64Type, Field, NullableDataType, Time64NanosecondType, ToByteSlice,
};

fn main() {
// Primitive Arrays
Expand Down Expand Up @@ -100,7 +102,7 @@ fn main() {

// Construct a list array from the above two
let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
DataType::List(Box::new(NullableDataType::new(DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.len(3)
.add_buffer(value_offsets)
Expand Down
4 changes: 2 additions & 2 deletions rust/arrow/src/array/array_binary.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -596,7 +596,7 @@ impl Array for DecimalArray {

#[cfg(test)]
mod tests {
use crate::datatypes::Field;
use crate::datatypes::NullableDataType;

use super::*;

Expand Down Expand Up @@ -908,7 +908,7 @@ mod tests {
.build();

let array_data = ArrayData::builder(DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Binary, false)),
Box::new(NullableDataType::new(DataType::Binary, false)),
4,
))
.len(3)
Expand Down
31 changes: 14 additions & 17 deletions rust/arrow/src/array/array_list.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,15 +297,12 @@ impl fmt::Debug for FixedSizeListArray {
#[cfg(test)]
mod tests {
use crate::{
array::ArrayData,
array::Int32Array,
buffer::Buffer,
datatypes::{Field, ToByteSlice},
memory,
util::bit_util,
array::ArrayData, array::Int32Array, buffer::Buffer, datatypes::ToByteSlice,
memory, util::bit_util,
};

use super::*;
use crate::datatypes::NullableDataType;

#[test]
fn test_list_array() {
Expand All @@ -321,7 +318,7 @@ mod tests {

// Construct a list array from the above two
let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
DataType::List(Box::new(NullableDataType::new(DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type.clone())
.len(3)
.add_buffer(value_offsets.clone())
Expand Down Expand Up @@ -391,7 +388,7 @@ mod tests {

// Construct a list array from the above two
let list_data_type =
DataType::LargeList(Box::new(Field::new("item", DataType::Int32, false)));
DataType::LargeList(Box::new(NullableDataType::new(DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type.clone())
.len(3)
.add_buffer(value_offsets.clone())
Expand Down Expand Up @@ -457,7 +454,7 @@ mod tests {

// Construct a list array from the above two
let list_data_type = DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Int32, false)),
Box::new(NullableDataType::new(DataType::Int32, false)),
3,
);
let list_data = ArrayData::builder(list_data_type.clone())
Expand Down Expand Up @@ -526,7 +523,7 @@ mod tests {

// Construct a list array from the above two
let list_data_type = DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Int32, false)),
Box::new(NullableDataType::new(DataType::Int32, false)),
3,
);
let list_data = ArrayData::builder(list_data_type)
Expand Down Expand Up @@ -560,7 +557,7 @@ mod tests {

// Construct a list array from the above two
let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
DataType::List(Box::new(NullableDataType::new(DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.len(9)
.add_buffer(value_offsets)
Expand Down Expand Up @@ -625,7 +622,7 @@ mod tests {

// Construct a list array from the above two
let list_data_type =
DataType::LargeList(Box::new(Field::new("item", DataType::Int32, false)));
DataType::LargeList(Box::new(NullableDataType::new(DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.len(9)
.add_buffer(value_offsets)
Expand Down Expand Up @@ -688,7 +685,7 @@ mod tests {

// Construct a fixed size list array from the above two
let list_data_type = DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Int32, false)),
Box::new(NullableDataType::new(DataType::Int32, false)),
2,
);
let list_data = ArrayData::builder(list_data_type)
Expand Down Expand Up @@ -739,7 +736,7 @@ mod tests {
.add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
.build();
let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
DataType::List(Box::new(NullableDataType::new(DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.len(3)
.add_child_data(value_data)
Expand All @@ -754,7 +751,7 @@ mod tests {
fn test_list_array_invalid_child_array_len() {
let value_offsets = Buffer::from(&[0, 2, 5, 7].to_byte_slice());
let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
DataType::List(Box::new(NullableDataType::new(DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.len(3)
.add_buffer(value_offsets)
Expand All @@ -773,7 +770,7 @@ mod tests {
let value_offsets = Buffer::from(&[2, 2, 5, 7].to_byte_slice());

let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
DataType::List(Box::new(NullableDataType::new(DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.len(3)
.add_buffer(value_offsets)
Expand Down Expand Up @@ -805,7 +802,7 @@ mod tests {
.build();

let list_data_type =
DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
DataType::List(Box::new(NullableDataType::new(DataType::Int32, false)));
let list_data = ArrayData::builder(list_data_type)
.add_buffer(buf2)
.add_child_data(value_data)
Expand Down
49 changes: 23 additions & 26 deletions rust/arrow/src/array/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -764,8 +764,7 @@ where
///
/// This is used for validating array data types in `append_data`
fn data_type(&self) -> DataType {
DataType::List(Box::new(Field::new(
"item",
DataType::List(Box::new(NullableDataType::new(
self.values_builder.data_type(),
true,
)))
Expand Down Expand Up @@ -834,8 +833,7 @@ where
let null_bit_buffer = self.bitmap_builder.finish();
let nulls = null_bit_buffer.count_set_bits();
self.offsets_builder.append(0).unwrap();
let data = ArrayData::builder(DataType::List(Box::new(Field::new(
"item",
let data = ArrayData::builder(DataType::List(Box::new(NullableDataType::new(
values_data.data_type().clone(),
true, // TODO: find a consistent way of getting this
))))
Expand Down Expand Up @@ -976,8 +974,7 @@ where
///
/// This is used for validating array data types in `append_data`
fn data_type(&self) -> DataType {
DataType::LargeList(Box::new(Field::new(
"item",
DataType::LargeList(Box::new(NullableDataType::new(
self.values_builder.data_type(),
true,
)))
Expand Down Expand Up @@ -1046,11 +1043,9 @@ where
let null_bit_buffer = self.bitmap_builder.finish();
let nulls = null_bit_buffer.count_set_bits();
self.offsets_builder.append(0).unwrap();
let data = ArrayData::builder(DataType::LargeList(Box::new(Field::new(
"item",
values_data.data_type().clone(),
true,
))))
let data = ArrayData::builder(DataType::LargeList(Box::new(
NullableDataType::new(values_data.data_type().clone(), true),
)))
.len(len)
.null_count(len - nulls)
.add_buffer(offset_buffer)
Expand Down Expand Up @@ -1158,7 +1153,7 @@ where
/// This is used for validating array data types in `append_data`
fn data_type(&self) -> DataType {
DataType::FixedSizeList(
Box::new(Field::new("item", self.values_builder.data_type(), true)),
Box::new(NullableDataType::new(self.values_builder.data_type(), true)),
self.list_len,
)
}
Expand Down Expand Up @@ -1237,7 +1232,7 @@ where
let null_bit_buffer = self.bitmap_builder.finish();
let nulls = null_bit_buffer.count_set_bits();
let data = ArrayData::builder(DataType::FixedSizeList(
Box::new(Field::new("item", values_data.data_type().clone(), true)),
Box::new(NullableDataType::new(values_data.data_type().clone(), true)),
self.list_len,
))
.len(len)
Expand Down Expand Up @@ -1458,7 +1453,10 @@ fn append_binary_data(
)) as ArrayDataRef;

Arc::new(ArrayData::new(
DataType::List(Box::new(Field::new("item", DataType::UInt8, true))),
DataType::List(Box::new(NullableDataType::new(
DataType::UInt8,
true,
))),
array.len(),
None,
array.null_buffer().cloned(),
Expand Down Expand Up @@ -1510,8 +1508,7 @@ fn append_large_binary_data(
)) as ArrayDataRef;

Arc::new(ArrayData::new(
DataType::LargeList(Box::new(Field::new(
"item",
DataType::LargeList(Box::new(NullableDataType::new(
DataType::UInt8,
true,
))),
Expand Down Expand Up @@ -1613,7 +1610,7 @@ impl ArrayBuilder for FixedSizeBinaryBuilder {
)) as ArrayDataRef;
let list_data = Arc::new(ArrayData::new(
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::UInt8, true)),
Box::new(NullableDataType::new(DataType::UInt8, true)),
self.builder.list_len,
),
array.len(),
Expand Down Expand Up @@ -1699,7 +1696,7 @@ impl ArrayBuilder for DecimalBuilder {
)) as ArrayDataRef;
let list_data = Arc::new(ArrayData::new(
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::UInt8, true)),
Box::new(NullableDataType::new(DataType::UInt8, true)),
self.builder.list_len,
),
array.len(),
Expand Down Expand Up @@ -3820,13 +3817,13 @@ mod tests {

#[test]
#[should_panic(
expected = "Data type List(Field { name: \"item\", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false }) is not currently supported"
expected = "Data type List(NullableDataType { data_type: Int64, nullable: true }) is not currently supported"
)]
fn test_struct_array_builder_from_schema_unsupported_type() {
let mut fields = Vec::new();
fields.push(Field::new("f1", DataType::Int16, false));
let list_type =
DataType::List(Box::new(Field::new("item", DataType::Int64, true)));
DataType::List(Box::new(NullableDataType::new(DataType::Int64, true)));
fields.push(Field::new("f2", list_type, false));

let _ = StructBuilder::from_fields(fields, 5);
Expand Down Expand Up @@ -4125,7 +4122,7 @@ mod tests {
let list_value_offsets =
Buffer::from(&[0, 3, 5, 11, 13, 13, 15, 15, 17].to_byte_slice());
let expected_list_data = ArrayData::new(
DataType::List(Box::new(Field::new("item", DataType::Int64, true))),
DataType::List(Box::new(NullableDataType::new(DataType::Int64, true))),
8,
None,
None,
Expand Down Expand Up @@ -4211,7 +4208,7 @@ mod tests {
&[0, 3, 5, 5, 13, 15, 15, 15, 19, 19, 19, 19, 23].to_byte_slice(),
);
let expected_list_data = ArrayData::new(
DataType::List(Box::new(Field::new("item", DataType::Int64, true))),
DataType::List(Box::new(NullableDataType::new(DataType::Int64, true))),
12,
None,
None,
Expand Down Expand Up @@ -4253,7 +4250,7 @@ mod tests {
]);
let list_value_offsets = Buffer::from(&[0, 2, 3, 6].to_byte_slice());
let list_data = ArrayData::new(
DataType::List(Box::new(Field::new("item", DataType::Utf8, true))),
DataType::List(Box::new(NullableDataType::new(DataType::Utf8, true))),
3,
None,
None,
Expand Down Expand Up @@ -4288,7 +4285,7 @@ mod tests {
]);
let list_value_offsets = Buffer::from(&[0, 2, 2, 4, 5, 8, 9, 12].to_byte_slice());
let expected_list_data = ArrayData::new(
DataType::List(Box::new(Field::new("item", DataType::Utf8, true))),
DataType::List(Box::new(NullableDataType::new(DataType::Utf8, true))),
7,
None,
None, // is this correct?
Expand Down Expand Up @@ -4377,7 +4374,7 @@ mod tests {
]);
let expected_list_data = ArrayData::new(
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::UInt16, true)),
Box::new(NullableDataType::new(DataType::UInt16, true)),
2,
),
12,
Expand Down Expand Up @@ -4450,7 +4447,7 @@ mod tests {
]);
let expected_list_data = ArrayData::new(
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::UInt8, true)),
Box::new(NullableDataType::new(DataType::UInt8, true)),
2,
),
12,
Expand Down
Loading