From fe2cda5c8fa5cc734326f9412ef67738db6bf6c3 Mon Sep 17 00:00:00 2001 From: Paddy Horan Date: Tue, 5 Feb 2019 22:38:06 -0500 Subject: [PATCH 1/6] Added (SIMD) bitwise AND/OR for `Buffer` --- rust/arrow/Cargo.toml | 1 + rust/arrow/src/compute/boolean_kernels.rs | 145 ++++++++++++++++++++++ rust/arrow/src/compute/mod.rs | 18 +++ rust/arrow/src/lib.rs | 1 + rust/arrow/src/mod.rs | 1 + 5 files changed, 166 insertions(+) create mode 100644 rust/arrow/src/compute/boolean_kernels.rs create mode 100644 rust/arrow/src/compute/mod.rs diff --git a/rust/arrow/Cargo.toml b/rust/arrow/Cargo.toml index 1ebd4e6ba10..6bc15f102e5 100644 --- a/rust/arrow/Cargo.toml +++ b/rust/arrow/Cargo.toml @@ -45,6 +45,7 @@ csv = "1.0.0" num = "0.2" regex = "1.1" lazy_static = "1.2" +packed_simd = "0.3.1" [dev-dependencies] criterion = "0.2" diff --git a/rust/arrow/src/compute/boolean_kernels.rs b/rust/arrow/src/compute/boolean_kernels.rs new file mode 100644 index 00000000000..2518782f6e5 --- /dev/null +++ b/rust/arrow/src/compute/boolean_kernels.rs @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use packed_simd::u8x64; +use std::slice::{from_raw_parts, from_raw_parts_mut}; + +use crate::buffer::{Buffer, MutableBuffer}; +use crate::builder::{BufferBuilderTrait, UInt8BufferBuilder}; + +/// SIMD accelerated version of bitwise binary operation for two `Buffer`'s. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +fn bitwise_bin_op_simd(left: &Buffer, right: &Buffer, op: F) -> Buffer +where + F: Fn(u8x64, u8x64) -> u8x64, +{ + let mut result = MutableBuffer::new(left.len()).with_bitset(left.len(), false); + let lanes = u8x64::lanes(); + for i in (0..left.len()).step_by(lanes) { + let left_data = + unsafe { from_raw_parts(left.raw_data().offset(i as isize), lanes) }; + let left_simd = unsafe { u8x64::from_slice_unaligned_unchecked(left_data) }; + let right_data = + unsafe { from_raw_parts(right.raw_data().offset(i as isize), lanes) }; + let right_simd = unsafe { u8x64::from_slice_unaligned_unchecked(right_data) }; + let simd_result = op(left_simd, right_simd); + let result_slice: &mut [u8] = unsafe { + from_raw_parts_mut( + (result.data_mut().as_mut_ptr() as *mut u8).offset(i as isize), + lanes, + ) + }; + unsafe { simd_result.write_to_slice_unaligned_unchecked(result_slice) }; + } + result.freeze() +} + +/// Default version of bitwise binary operation for two `Buffer`'s where SIMD is not +/// available. +fn bitwise_bin_op_default(left: &Buffer, right: &Buffer, op: F) -> Buffer +where + F: Fn(&u8, &u8) -> u8, +{ + let mut builder = UInt8BufferBuilder::new(left.len()); + for i in 0..left.len() { + unsafe { + builder + .append(op( + left.data().get_unchecked(i), + right.data().get_unchecked(i), + )) + .unwrap(); + } + } + builder.finish() +} + +/// Bitwise binary AND for two `Buffer`'s. +pub fn bitwise_and(left: &Buffer, right: &Buffer) -> Buffer { + assert_eq!(left.len(), right.len(), "Buffers must be the same size."); + + // SIMD implementation if available + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + return bitwise_bin_op_simd(&left, &right, |a, b| a & b); + + // Default implementation + #[allow(unreachable_code)] + bitwise_bin_op_default(&left, &right, |a, b| a & b) +} + +/// Bitwise binary OR for two `Buffer`'s. +pub fn bitwise_or(left: &Buffer, right: &Buffer) -> Buffer { + assert_eq!(left.len(), right.len(), "Buffers must be the same size."); + + // SIMD implementation if available + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + return bitwise_bin_op_simd(&left, &right, |a, b| a | b); + + // Default implementation + #[allow(unreachable_code)] + bitwise_bin_op_default(&left, &right, |a, b| a | b) +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_bitwise_and() { + let buf1 = Buffer::from([0b01101010]); + let buf2 = Buffer::from([0b01001110]); + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + assert_eq!( + Buffer::from([0b01001010]), + bitwise_bin_op_simd(&buf1, &buf2, |a, b| a & b) + ); + } + assert_eq!( + Buffer::from([0b01001010]), + bitwise_bin_op_default(&buf1, &buf2, |a, b| a & b) + ); + assert_eq!(Buffer::from([0b01001010]), bitwise_and(&buf1, &buf2)); + } + + #[test] + fn test_bitwise_or() { + let buf1 = Buffer::from([0b01101010]); + let buf2 = Buffer::from([0b01001110]); + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + assert_eq!( + Buffer::from([0b01101110]), + bitwise_bin_op_simd(&buf1, &buf2, |a, b| a | b) + ); + } + assert_eq!( + Buffer::from([0b01101110]), + bitwise_bin_op_default(&buf1, &buf2, |a, b| a | b) + ); + assert_eq!(Buffer::from([0b01101110]), bitwise_or(&buf1, &buf2)); + } + + #[test] + #[should_panic(expected = "Buffers must be the same size.")] + fn test_buffer_bitand_different_sizes() { + let buf1 = Buffer::from([1_u8, 1_u8]); + let buf2 = Buffer::from([0b01001110]); + let _buf3 = bitwise_and(&buf1, &buf2); + } +} diff --git a/rust/arrow/src/compute/mod.rs b/rust/arrow/src/compute/mod.rs new file mode 100644 index 00000000000..c79f513fe9f --- /dev/null +++ b/rust/arrow/src/compute/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod boolean_kernels; diff --git a/rust/arrow/src/lib.rs b/rust/arrow/src/lib.rs index dbac4db1151..5608a1f3620 100644 --- a/rust/arrow/src/lib.rs +++ b/rust/arrow/src/lib.rs @@ -34,6 +34,7 @@ pub mod array_ops; pub mod bitmap; pub mod buffer; pub mod builder; +pub mod compute; pub mod csv; pub mod datatypes; pub mod error; diff --git a/rust/arrow/src/mod.rs b/rust/arrow/src/mod.rs index b9fa43ab818..7a98c6ab5c1 100644 --- a/rust/arrow/src/mod.rs +++ b/rust/arrow/src/mod.rs @@ -20,6 +20,7 @@ pub mod array_data; pub mod bitmap; pub mod buffer; pub mod builder; +pub mod compute; pub mod csv; pub mod datatypes; pub mod error; From 6a731b7ef83470da1c3585dbbbd11e76ac8feea9 Mon Sep 17 00:00:00 2001 From: Paddy Horan Date: Tue, 5 Feb 2019 23:02:33 -0500 Subject: [PATCH 2/6] Added benchmark --- rust/arrow/Cargo.toml | 6 ++- rust/arrow/benches/boolean_kernels.rs | 54 +++++++++++++++++++++++ rust/arrow/src/compute/boolean_kernels.rs | 4 +- 3 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 rust/arrow/benches/boolean_kernels.rs diff --git a/rust/arrow/Cargo.toml b/rust/arrow/Cargo.toml index 6bc15f102e5..04e8ac05c35 100644 --- a/rust/arrow/Cargo.toml +++ b/rust/arrow/Cargo.toml @@ -57,4 +57,8 @@ harness = false [[bench]] name = "builder" -harness = false \ No newline at end of file +harness = false + +[[bench]] +name = "boolean_kernels" +harness = false diff --git a/rust/arrow/benches/boolean_kernels.rs b/rust/arrow/benches/boolean_kernels.rs new file mode 100644 index 00000000000..c440a26b09b --- /dev/null +++ b/rust/arrow/benches/boolean_kernels.rs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#[macro_use] +extern crate criterion; +use criterion::Criterion; + +extern crate arrow; + +use arrow::buffer::Buffer; +use arrow::builder::{BufferBuilderTrait, UInt8BufferBuilder}; +use arrow::compute::boolean_kernels::*; + +fn create_buffer(size: usize) -> Buffer { + let mut builder = UInt8BufferBuilder::new(size); + for _i in 0..size { + builder.append(1_u8).unwrap(); + } + builder.finish() +} + +fn bitwise_and_default(size: usize) { + let buffer_a = create_buffer(size); + let buffer_b = create_buffer(size); + criterion::black_box(bitwise_bin_op_default(&buffer_a, &buffer_b, |a, b| a & b)); +} + +fn bitwise_and_simd(size: usize) { + let buffer_a = create_buffer(size); + let buffer_b = create_buffer(size); + criterion::black_box(bitwise_bin_op_simd(&buffer_a, &buffer_b, |a, b| a & b)); +} + +fn add_benchmark(c: &mut Criterion) { + c.bench_function("add", |b| b.iter(|| bitwise_and_default(512))); + c.bench_function("add simd", |b| b.iter(|| bitwise_and_simd(512))); +} + +criterion_group!(benches, add_benchmark); +criterion_main!(benches); diff --git a/rust/arrow/src/compute/boolean_kernels.rs b/rust/arrow/src/compute/boolean_kernels.rs index 2518782f6e5..a5f99125a63 100644 --- a/rust/arrow/src/compute/boolean_kernels.rs +++ b/rust/arrow/src/compute/boolean_kernels.rs @@ -23,7 +23,7 @@ use crate::builder::{BufferBuilderTrait, UInt8BufferBuilder}; /// SIMD accelerated version of bitwise binary operation for two `Buffer`'s. #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -fn bitwise_bin_op_simd(left: &Buffer, right: &Buffer, op: F) -> Buffer +pub fn bitwise_bin_op_simd(left: &Buffer, right: &Buffer, op: F) -> Buffer where F: Fn(u8x64, u8x64) -> u8x64, { @@ -50,7 +50,7 @@ where /// Default version of bitwise binary operation for two `Buffer`'s where SIMD is not /// available. -fn bitwise_bin_op_default(left: &Buffer, right: &Buffer, op: F) -> Buffer +pub fn bitwise_bin_op_default(left: &Buffer, right: &Buffer, op: F) -> Buffer where F: Fn(&u8, &u8) -> u8, { From 26d58db9da79798793a003461241c072b9cda0f7 Mon Sep 17 00:00:00 2001 From: Paddy Horan Date: Thu, 7 Feb 2019 22:42:40 -0500 Subject: [PATCH 3/6] Removed boolean kernels in favor of traits. --- rust/arrow/Cargo.toml | 2 +- .../{boolean_kernels.rs => bitwise_ops.rs} | 19 ++- rust/arrow/src/buffer.rs | 141 +++++++++++++++++ rust/arrow/src/compute/boolean_kernels.rs | 145 ------------------ rust/arrow/src/compute/mod.rs | 18 --- rust/arrow/src/lib.rs | 1 - rust/arrow/src/mod.rs | 1 - rust/arrow/src/util/bit_util.rs | 42 +++++ 8 files changed, 200 insertions(+), 169 deletions(-) rename rust/arrow/benches/{boolean_kernels.rs => bitwise_ops.rs} (77%) delete mode 100644 rust/arrow/src/compute/boolean_kernels.rs delete mode 100644 rust/arrow/src/compute/mod.rs diff --git a/rust/arrow/Cargo.toml b/rust/arrow/Cargo.toml index 04e8ac05c35..6e2d4839bba 100644 --- a/rust/arrow/Cargo.toml +++ b/rust/arrow/Cargo.toml @@ -60,5 +60,5 @@ name = "builder" harness = false [[bench]] -name = "boolean_kernels" +name = "bitwise_ops" harness = false diff --git a/rust/arrow/benches/boolean_kernels.rs b/rust/arrow/benches/bitwise_ops.rs similarity index 77% rename from rust/arrow/benches/boolean_kernels.rs rename to rust/arrow/benches/bitwise_ops.rs index c440a26b09b..0554e7b6f38 100644 --- a/rust/arrow/benches/boolean_kernels.rs +++ b/rust/arrow/benches/bitwise_ops.rs @@ -23,7 +23,6 @@ extern crate arrow; use arrow::buffer::Buffer; use arrow::builder::{BufferBuilderTrait, UInt8BufferBuilder}; -use arrow::compute::boolean_kernels::*; fn create_buffer(size: usize) -> Buffer { let mut builder = UInt8BufferBuilder::new(size); @@ -36,13 +35,27 @@ fn create_buffer(size: usize) -> Buffer { fn bitwise_and_default(size: usize) { let buffer_a = create_buffer(size); let buffer_b = create_buffer(size); - criterion::black_box(bitwise_bin_op_default(&buffer_a, &buffer_b, |a, b| a & b)); + + criterion::black_box({ + let mut builder = UInt8BufferBuilder::new(buffer_a.len()); + for i in 0..buffer_a.len() { + unsafe { + builder + .append( + buffer_a.data().get_unchecked(i) + & buffer_b.data().get_unchecked(i), + ) + .unwrap(); + } + } + builder.finish() + }); } fn bitwise_and_simd(size: usize) { let buffer_a = create_buffer(size); let buffer_b = create_buffer(size); - criterion::black_box(bitwise_bin_op_simd(&buffer_a, &buffer_b, |a, b| a & b)); + criterion::black_box(&buffer_a & &buffer_b); } fn add_benchmark(c: &mut Criterion) { diff --git a/rust/arrow/src/buffer.rs b/rust/arrow/src/buffer.rs index 70aea63aedc..5ff7d9e4eae 100644 --- a/rust/arrow/src/buffer.rs +++ b/rust/arrow/src/buffer.rs @@ -18,12 +18,17 @@ //! The main type in the module is `Buffer`, a contiguous immutable memory region of //! fixed size aligned at a 64-byte boundary. `MutableBuffer` is like `Buffer`, but it can //! be mutated and grown. +//! +use packed_simd::u8x64; use std::cmp; use std::io::{Error as IoError, ErrorKind, Result as IoResult, Write}; use std::mem; +use std::ops::{BitAnd, BitOr}; +use std::slice::{from_raw_parts, from_raw_parts_mut}; use std::sync::Arc; +use crate::builder::{BufferBuilderTrait, UInt8BufferBuilder}; use crate::error::Result; use crate::memory; use crate::util::bit_util; @@ -141,6 +146,120 @@ impl> From for Buffer { } } +impl<'a, 'b> BitAnd<&'b Buffer> for &'a Buffer { + type Output = Buffer; + + fn bitand(self, rhs: &'b Buffer) -> Buffer { + assert_eq!( + self.len(), + rhs.len(), + "Buffers must be the same size to apply Bitwise OR." + ); + + // SIMD implementation if available + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let mut result = + MutableBuffer::new(self.len()).with_bitset(self.len(), false); + let lanes = u8x64::lanes(); + for i in (0..self.len()).step_by(lanes) { + let left_data = + unsafe { from_raw_parts(self.raw_data().offset(i as isize), lanes) }; + let right_data = + unsafe { from_raw_parts(rhs.raw_data().offset(i as isize), lanes) }; + let result_slice: &mut [u8] = unsafe { + from_raw_parts_mut( + (result.data_mut().as_mut_ptr() as *mut u8).offset(i as isize), + lanes, + ) + }; + unsafe { + bit_util::bitwise_bin_op_simd( + &left_data, + &right_data, + result_slice, + |a, b| a & b, + ) + }; + } + return result.freeze(); + } + + // Default implementation + #[allow(unreachable_code)] + { + let mut builder = UInt8BufferBuilder::new(self.len()); + for i in 0..self.len() { + unsafe { + builder + .append( + self.data().get_unchecked(i) & rhs.data().get_unchecked(i), + ) + .unwrap(); + } + } + builder.finish() + } + } +} + +impl<'a, 'b> BitOr<&'b Buffer> for &'a Buffer { + type Output = Buffer; + + fn bitor(self, rhs: &'b Buffer) -> Buffer { + assert_eq!( + self.len(), + rhs.len(), + "Buffers must be the same size to apply Bitwise OR." + ); + + // SIMD implementation if available + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let mut result = + MutableBuffer::new(self.len()).with_bitset(self.len(), false); + let lanes = u8x64::lanes(); + for i in (0..self.len()).step_by(lanes) { + let left_data = + unsafe { from_raw_parts(self.raw_data().offset(i as isize), lanes) }; + let right_data = + unsafe { from_raw_parts(rhs.raw_data().offset(i as isize), lanes) }; + let result_slice: &mut [u8] = unsafe { + from_raw_parts_mut( + (result.data_mut().as_mut_ptr() as *mut u8).offset(i as isize), + lanes, + ) + }; + unsafe { + bit_util::bitwise_bin_op_simd( + &left_data, + &right_data, + result_slice, + |a, b| a | b, + ) + }; + } + return result.freeze(); + } + + // Default implementation + #[allow(unreachable_code)] + { + let mut builder = UInt8BufferBuilder::new(self.len()); + for i in 0..self.len() { + unsafe { + builder + .append( + self.data().get_unchecked(i) & rhs.data().get_unchecked(i), + ) + .unwrap(); + } + } + builder.finish() + } + } +} + unsafe impl Sync for Buffer {} unsafe impl Send for Buffer {} @@ -434,6 +553,28 @@ mod tests { assert_eq!(256, bit_util::count_set_bits(buf.data())); } + #[test] + fn test_bitwise_and() { + let buf1 = Buffer::from([0b01101010]); + let buf2 = Buffer::from([0b01001110]); + assert_eq!(Buffer::from([0b01001010]), &buf1 & &buf2); + } + + #[test] + fn test_bitwise_or() { + let buf1 = Buffer::from([0b01101010]); + let buf2 = Buffer::from([0b01001110]); + assert_eq!(Buffer::from([0b01101110]), &buf1 | &buf2); + } + + #[test] + #[should_panic(expected = "Buffers must be the same size to apply Bitwise OR.")] + fn test_buffer_bitand_different_sizes() { + let buf1 = Buffer::from([1_u8, 1_u8]); + let buf2 = Buffer::from([0b01001110]); + let _buf3 = &buf1 & &buf2; + } + #[test] fn test_mutable_new() { let buf = MutableBuffer::new(63); diff --git a/rust/arrow/src/compute/boolean_kernels.rs b/rust/arrow/src/compute/boolean_kernels.rs deleted file mode 100644 index a5f99125a63..00000000000 --- a/rust/arrow/src/compute/boolean_kernels.rs +++ /dev/null @@ -1,145 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use packed_simd::u8x64; -use std::slice::{from_raw_parts, from_raw_parts_mut}; - -use crate::buffer::{Buffer, MutableBuffer}; -use crate::builder::{BufferBuilderTrait, UInt8BufferBuilder}; - -/// SIMD accelerated version of bitwise binary operation for two `Buffer`'s. -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -pub fn bitwise_bin_op_simd(left: &Buffer, right: &Buffer, op: F) -> Buffer -where - F: Fn(u8x64, u8x64) -> u8x64, -{ - let mut result = MutableBuffer::new(left.len()).with_bitset(left.len(), false); - let lanes = u8x64::lanes(); - for i in (0..left.len()).step_by(lanes) { - let left_data = - unsafe { from_raw_parts(left.raw_data().offset(i as isize), lanes) }; - let left_simd = unsafe { u8x64::from_slice_unaligned_unchecked(left_data) }; - let right_data = - unsafe { from_raw_parts(right.raw_data().offset(i as isize), lanes) }; - let right_simd = unsafe { u8x64::from_slice_unaligned_unchecked(right_data) }; - let simd_result = op(left_simd, right_simd); - let result_slice: &mut [u8] = unsafe { - from_raw_parts_mut( - (result.data_mut().as_mut_ptr() as *mut u8).offset(i as isize), - lanes, - ) - }; - unsafe { simd_result.write_to_slice_unaligned_unchecked(result_slice) }; - } - result.freeze() -} - -/// Default version of bitwise binary operation for two `Buffer`'s where SIMD is not -/// available. -pub fn bitwise_bin_op_default(left: &Buffer, right: &Buffer, op: F) -> Buffer -where - F: Fn(&u8, &u8) -> u8, -{ - let mut builder = UInt8BufferBuilder::new(left.len()); - for i in 0..left.len() { - unsafe { - builder - .append(op( - left.data().get_unchecked(i), - right.data().get_unchecked(i), - )) - .unwrap(); - } - } - builder.finish() -} - -/// Bitwise binary AND for two `Buffer`'s. -pub fn bitwise_and(left: &Buffer, right: &Buffer) -> Buffer { - assert_eq!(left.len(), right.len(), "Buffers must be the same size."); - - // SIMD implementation if available - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - return bitwise_bin_op_simd(&left, &right, |a, b| a & b); - - // Default implementation - #[allow(unreachable_code)] - bitwise_bin_op_default(&left, &right, |a, b| a & b) -} - -/// Bitwise binary OR for two `Buffer`'s. -pub fn bitwise_or(left: &Buffer, right: &Buffer) -> Buffer { - assert_eq!(left.len(), right.len(), "Buffers must be the same size."); - - // SIMD implementation if available - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - return bitwise_bin_op_simd(&left, &right, |a, b| a | b); - - // Default implementation - #[allow(unreachable_code)] - bitwise_bin_op_default(&left, &right, |a, b| a | b) -} - -#[cfg(test)] -mod tests { - - use super::*; - - #[test] - fn test_bitwise_and() { - let buf1 = Buffer::from([0b01101010]); - let buf2 = Buffer::from([0b01001110]); - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - assert_eq!( - Buffer::from([0b01001010]), - bitwise_bin_op_simd(&buf1, &buf2, |a, b| a & b) - ); - } - assert_eq!( - Buffer::from([0b01001010]), - bitwise_bin_op_default(&buf1, &buf2, |a, b| a & b) - ); - assert_eq!(Buffer::from([0b01001010]), bitwise_and(&buf1, &buf2)); - } - - #[test] - fn test_bitwise_or() { - let buf1 = Buffer::from([0b01101010]); - let buf2 = Buffer::from([0b01001110]); - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - assert_eq!( - Buffer::from([0b01101110]), - bitwise_bin_op_simd(&buf1, &buf2, |a, b| a | b) - ); - } - assert_eq!( - Buffer::from([0b01101110]), - bitwise_bin_op_default(&buf1, &buf2, |a, b| a | b) - ); - assert_eq!(Buffer::from([0b01101110]), bitwise_or(&buf1, &buf2)); - } - - #[test] - #[should_panic(expected = "Buffers must be the same size.")] - fn test_buffer_bitand_different_sizes() { - let buf1 = Buffer::from([1_u8, 1_u8]); - let buf2 = Buffer::from([0b01001110]); - let _buf3 = bitwise_and(&buf1, &buf2); - } -} diff --git a/rust/arrow/src/compute/mod.rs b/rust/arrow/src/compute/mod.rs deleted file mode 100644 index c79f513fe9f..00000000000 --- a/rust/arrow/src/compute/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod boolean_kernels; diff --git a/rust/arrow/src/lib.rs b/rust/arrow/src/lib.rs index 5608a1f3620..dbac4db1151 100644 --- a/rust/arrow/src/lib.rs +++ b/rust/arrow/src/lib.rs @@ -34,7 +34,6 @@ pub mod array_ops; pub mod bitmap; pub mod buffer; pub mod builder; -pub mod compute; pub mod csv; pub mod datatypes; pub mod error; diff --git a/rust/arrow/src/mod.rs b/rust/arrow/src/mod.rs index 7a98c6ab5c1..b9fa43ab818 100644 --- a/rust/arrow/src/mod.rs +++ b/rust/arrow/src/mod.rs @@ -20,7 +20,6 @@ pub mod array_data; pub mod bitmap; pub mod buffer; pub mod builder; -pub mod compute; pub mod csv; pub mod datatypes; pub mod error; diff --git a/rust/arrow/src/util/bit_util.rs b/rust/arrow/src/util/bit_util.rs index 4674783b092..23bc3d5bb22 100644 --- a/rust/arrow/src/util/bit_util.rs +++ b/rust/arrow/src/util/bit_util.rs @@ -17,6 +17,8 @@ //! Utils for working with bits +use packed_simd::u8x64; + static BIT_MASK: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128]; static POPCOUNT_TABLE: [u8; 256] = [ @@ -117,6 +119,22 @@ pub fn ceil(value: usize, divisor: usize) -> usize { result } +/// Performs SIMD bitwise binary operations. +/// +/// Note that each slice should be 64 bytes and it is the callers responsibility to ensure that +/// this is the case. If passed slices larger than 64 bytes the operation will only be performed +/// on the first 64 bytes. Slices less than 64 bytes will panic. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +pub unsafe fn bitwise_bin_op_simd(left: &[u8], right: &[u8], result: &mut [u8], op: F) +where + F: Fn(u8x64, u8x64) -> u8x64, +{ + let left_simd = u8x64::from_slice_unaligned_unchecked(left); + let right_simd = u8x64::from_slice_unaligned_unchecked(right); + let simd_result = op(left_simd, right_simd); + simd_result.write_to_slice_unaligned_unchecked(result); +} + #[cfg(test)] mod tests { use rand::{thread_rng, Rng}; @@ -270,4 +288,28 @@ mod tests { assert_eq!(ceil(10, 10000000000), 1); assert_eq!(ceil(10000000000, 1000000000), 10); } + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[test] + fn test_bitwise_and_simd() { + let buf1 = [0b00110011u8; 64]; + let buf2 = [0b11110000u8; 64]; + let mut buf3 = [0b00000000; 64]; + unsafe { bitwise_bin_op_simd(&buf1, &buf2, &mut buf3, |a, b| a & b) }; + for i in buf3.iter() { + assert_eq!(&0b00110000u8, i); + } + } + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[test] + fn test_bitwise_or_simd() { + let buf1 = [0b00110011u8; 64]; + let buf2 = [0b11110000u8; 64]; + let mut buf3 = [0b00000000; 64]; + unsafe { bitwise_bin_op_simd(&buf1, &buf2, &mut buf3, |a, b| a | b) }; + for i in buf3.iter() { + assert_eq!(&0b11110011u8, i); + } + } } From a8534b152409fb7624f42c3b6dbf6e0401346ecc Mon Sep 17 00:00:00 2001 From: Paddy Horan Date: Thu, 7 Feb 2019 22:52:19 -0500 Subject: [PATCH 4/6] Updated typo. --- rust/arrow/src/buffer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/arrow/src/buffer.rs b/rust/arrow/src/buffer.rs index 5ff7d9e4eae..193e51714b9 100644 --- a/rust/arrow/src/buffer.rs +++ b/rust/arrow/src/buffer.rs @@ -153,7 +153,7 @@ impl<'a, 'b> BitAnd<&'b Buffer> for &'a Buffer { assert_eq!( self.len(), rhs.len(), - "Buffers must be the same size to apply Bitwise OR." + "Buffers must be the same size to apply Bitwise AND." ); // SIMD implementation if available @@ -572,7 +572,7 @@ mod tests { fn test_buffer_bitand_different_sizes() { let buf1 = Buffer::from([1_u8, 1_u8]); let buf2 = Buffer::from([0b01001110]); - let _buf3 = &buf1 & &buf2; + let _buf3 = &buf1 | &buf2; } #[test] From 07ccbc5126cc3dc84a1e2e08ddeaebb724c9a10a Mon Sep 17 00:00:00 2001 From: Paddy Horan Date: Sat, 9 Feb 2019 13:21:05 -0500 Subject: [PATCH 5/6] Cleaned up code. --- rust/arrow/src/buffer.rs | 78 +++++++++++++++------------------------- 1 file changed, 29 insertions(+), 49 deletions(-) diff --git a/rust/arrow/src/buffer.rs b/rust/arrow/src/buffer.rs index 193e51714b9..447a5753d2b 100644 --- a/rust/arrow/src/buffer.rs +++ b/rust/arrow/src/buffer.rs @@ -146,6 +146,32 @@ impl> From for Buffer { } } +/// Helper function for SIMD `BitAnd` and `BitOr` implementations +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +fn bitwise_bin_op_simd_helper(left: &Buffer, right: &Buffer, op: F) -> Buffer +where + F: Fn(u8x64, u8x64) -> u8x64, +{ + let mut result = MutableBuffer::new(left.len()).with_bitset(left.len(), false); + let lanes = u8x64::lanes(); + for i in (0..left.len()).step_by(lanes) { + let left_data = + unsafe { from_raw_parts(left.raw_data().offset(i as isize), lanes) }; + let right_data = + unsafe { from_raw_parts(right.raw_data().offset(i as isize), lanes) }; + let result_slice: &mut [u8] = unsafe { + from_raw_parts_mut( + (result.data_mut().as_mut_ptr() as *mut u8).offset(i as isize), + lanes, + ) + }; + unsafe { + bit_util::bitwise_bin_op_simd(&left_data, &right_data, result_slice, &op) + }; + } + return result.freeze(); +} + impl<'a, 'b> BitAnd<&'b Buffer> for &'a Buffer { type Output = Buffer; @@ -159,30 +185,7 @@ impl<'a, 'b> BitAnd<&'b Buffer> for &'a Buffer { // SIMD implementation if available #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { - let mut result = - MutableBuffer::new(self.len()).with_bitset(self.len(), false); - let lanes = u8x64::lanes(); - for i in (0..self.len()).step_by(lanes) { - let left_data = - unsafe { from_raw_parts(self.raw_data().offset(i as isize), lanes) }; - let right_data = - unsafe { from_raw_parts(rhs.raw_data().offset(i as isize), lanes) }; - let result_slice: &mut [u8] = unsafe { - from_raw_parts_mut( - (result.data_mut().as_mut_ptr() as *mut u8).offset(i as isize), - lanes, - ) - }; - unsafe { - bit_util::bitwise_bin_op_simd( - &left_data, - &right_data, - result_slice, - |a, b| a & b, - ) - }; - } - return result.freeze(); + return bitwise_bin_op_simd_helper(&self, &rhs, |a, b| a & b); } // Default implementation @@ -216,30 +219,7 @@ impl<'a, 'b> BitOr<&'b Buffer> for &'a Buffer { // SIMD implementation if available #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { - let mut result = - MutableBuffer::new(self.len()).with_bitset(self.len(), false); - let lanes = u8x64::lanes(); - for i in (0..self.len()).step_by(lanes) { - let left_data = - unsafe { from_raw_parts(self.raw_data().offset(i as isize), lanes) }; - let right_data = - unsafe { from_raw_parts(rhs.raw_data().offset(i as isize), lanes) }; - let result_slice: &mut [u8] = unsafe { - from_raw_parts_mut( - (result.data_mut().as_mut_ptr() as *mut u8).offset(i as isize), - lanes, - ) - }; - unsafe { - bit_util::bitwise_bin_op_simd( - &left_data, - &right_data, - result_slice, - |a, b| a | b, - ) - }; - } - return result.freeze(); + return bitwise_bin_op_simd_helper(&self, &rhs, |a, b| a | b); } // Default implementation @@ -250,7 +230,7 @@ impl<'a, 'b> BitOr<&'b Buffer> for &'a Buffer { unsafe { builder .append( - self.data().get_unchecked(i) & rhs.data().get_unchecked(i), + self.data().get_unchecked(i) | rhs.data().get_unchecked(i), ) .unwrap(); } From 3476ace1ea62958d0a9fb9ef1fed9a18663c72c6 Mon Sep 17 00:00:00 2001 From: Paddy Horan Date: Sat, 9 Feb 2019 13:30:27 -0500 Subject: [PATCH 6/6] Updated benchmark to include bitwise OR op --- rust/arrow/benches/bitwise_ops.rs | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/rust/arrow/benches/bitwise_ops.rs b/rust/arrow/benches/bitwise_ops.rs index 0554e7b6f38..434ff4d9508 100644 --- a/rust/arrow/benches/bitwise_ops.rs +++ b/rust/arrow/benches/bitwise_ops.rs @@ -32,7 +32,10 @@ fn create_buffer(size: usize) -> Buffer { builder.finish() } -fn bitwise_and_default(size: usize) { +fn bitwise_default(size: usize, op: F) +where + F: Fn(&u8, &u8) -> u8, +{ let buffer_a = create_buffer(size); let buffer_b = create_buffer(size); @@ -41,10 +44,10 @@ fn bitwise_and_default(size: usize) { for i in 0..buffer_a.len() { unsafe { builder - .append( - buffer_a.data().get_unchecked(i) - & buffer_b.data().get_unchecked(i), - ) + .append(op( + buffer_a.data().get_unchecked(i), + buffer_b.data().get_unchecked(i), + )) .unwrap(); } } @@ -52,15 +55,20 @@ fn bitwise_and_default(size: usize) { }); } -fn bitwise_and_simd(size: usize) { +fn bitwise_simd(size: usize, op: F) +where + F: Fn(&Buffer, &Buffer) -> Buffer, +{ let buffer_a = create_buffer(size); let buffer_b = create_buffer(size); - criterion::black_box(&buffer_a & &buffer_b); + criterion::black_box(op(&buffer_a, &buffer_b)); } fn add_benchmark(c: &mut Criterion) { - c.bench_function("add", |b| b.iter(|| bitwise_and_default(512))); - c.bench_function("add simd", |b| b.iter(|| bitwise_and_simd(512))); + c.bench_function("add", |b| b.iter(|| bitwise_default(512, |a, b| a & b))); + c.bench_function("add simd", |b| b.iter(|| bitwise_simd(512, |a, b| a & b))); + c.bench_function("or", |b| b.iter(|| bitwise_default(512, |a, b| a | b))); + c.bench_function("or simd", |b| b.iter(|| bitwise_simd(512, |a, b| a | b))); } criterion_group!(benches, add_benchmark);