PARQUET-1104: Upgrade to Apache Arrow 0.7.0 RC0

CMakeLists.txt

@@ -499,7 +499,8 @@ if (PARQUET_BOOST_USE_SHARED)
     boost_shared_regex)
 else()
   set(BOOST_LINK_LIBS
-    boost_static_regex)
+    boost_static_regex
+    boost_static_system)
 endif()
 
 #############################################################

README.md

@@ -66,7 +66,8 @@ Check [Windows developer guide][1] for instructions to build parquet-cpp on Wind
 
 ## Third Party Dependencies
 
-- Apache Arrow (memory management, compression, IO, optional columnar data adapters)
+- Apache Arrow >= 0.7.0 (memory management, compression, IO, optional columnar
+  data adapters)
 - Thrift 0.7+ [install instructions](https://thrift.apache.org/docs/install/)
 - googletest 1.7.0 (cannot be installed with package managers)
 - Google Benchmark (only required if building benchmarks)

cmake_modules/ThirdpartyToolchain.cmake

@@ -70,20 +70,24 @@ if (PARQUET_BOOST_USE_SHARED)
     add_definitions(-DBOOST_ALL_DYN_LINK)
   endif()
 
-  find_package(Boost COMPONENTS regex REQUIRED)
+  find_package(Boost COMPONENTS regex system REQUIRED)
   if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG")
     set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
+    set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
   else()
     set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
+    set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
   endif()
 else()
   # Find static Boost libraries.
   set(Boost_USE_STATIC_LIBS ON)
-  find_package(Boost COMPONENTS regex REQUIRED)
+  find_package(Boost COMPONENTS regex system REQUIRED)
   if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG")
     set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
+    set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
   else()
     set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
+    set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
   endif()
 endif()
 
@@ -98,9 +102,19 @@ if (PARQUET_BOOST_USE_SHARED)
     set_target_properties(boost_shared_regex
                           PROPERTIES IMPORTED_LOCATION "${BOOST_SHARED_REGEX_LIBRARY}")
   endif()
+  add_library(boost_shared_system SHARED IMPORTED)
+  if (MSVC)
+    set_target_properties(boost_shared_system
+                          PROPERTIES IMPORTED_IMPLIB "${BOOST_SHARED_SYSTEM_LIBRARY}")
+  else()
+    set_target_properties(boost_shared_system
+                          PROPERTIES IMPORTED_LOCATION "${BOOST_SHARED_SYSTEM_LIBRARY}")
+  endif()
 else()
   add_library(boost_static_regex STATIC IMPORTED)
   set_target_properties(boost_static_regex PROPERTIES IMPORTED_LOCATION ${BOOST_STATIC_REGEX_LIBRARY})
+  add_library(boost_static_system STATIC IMPORTED)
+  set_target_properties(boost_static_system PROPERTIES IMPORTED_LOCATION ${BOOST_STATIC_SYSTEM_LIBRARY})
 endif()
 
 include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
@@ -339,7 +353,7 @@ if (NOT ARROW_FOUND)
     -DARROW_BUILD_TESTS=OFF)
 
   if ("$ENV{PARQUET_ARROW_VERSION}" STREQUAL "")
-    set(ARROW_VERSION "939957f33ed0dd02013917b366ff85eb857c3947")
+    set(ARROW_VERSION "97f9029ce835dfc2655ca91b9820a2e6aed89107")
   else()
     set(ARROW_VERSION "$ENV{PARQUET_ARROW_VERSION}")
   endif()

src/parquet/arrow/arrow-reader-writer-test.cc

@@ -779,10 +779,10 @@ TEST_F(TestUInt32ParquetIO, Parquet_1_0_Compability) {
     }
   }
 
-  const int32_t kOffset = 0;
-  ASSERT_OK(MakePrimitiveArray(std::make_shared<::arrow::Int64Type>(), values->length(),
-                               int64_data, values->null_bitmap(), values->null_count(),
-                               kOffset, &expected_values));
+  std::vector<std::shared_ptr<Buffer>> buffers{values->null_bitmap(), int64_data};
+  auto arr_data = std::make_shared<::arrow::ArrayData>(::arrow::int64(), values->length(),
+                                                       buffers, values->null_count());
+  ASSERT_OK(MakeArray(arr_data, &expected_values));
   this->ReadAndCheckSingleColumnTable(expected_values);
 }
 

src/parquet/arrow/reader.cc

@@ -334,7 +334,7 @@ class PARQUET_NO_EXPORT StructImpl : public ColumnReader::Impl {
   std::shared_ptr<Field> field_;
   PoolBuffer def_levels_buffer_;
 
-  Status DefLevelsToNullArray(std::shared_ptr<MutableBuffer>* null_bitmap,
+  Status DefLevelsToNullArray(std::shared_ptr<Buffer>* null_bitmap,
                               int64_t* null_count);
   void InitField(const NodePtr& node, const std::vector<std::shared_ptr<Impl>>& children);
 };
@@ -1376,9 +1376,9 @@ Status ColumnReader::NextBatch(int batch_size, std::shared_ptr<Array>* out) {
 
 // StructImpl methods
 
-Status StructImpl::DefLevelsToNullArray(std::shared_ptr<MutableBuffer>* null_bitmap_out,
+Status StructImpl::DefLevelsToNullArray(std::shared_ptr<Buffer>* null_bitmap_out,
                                         int64_t* null_count_out) {
-  std::shared_ptr<MutableBuffer> null_bitmap;
+  std::shared_ptr<Buffer> null_bitmap;
   auto null_count = 0;
   ValueLevelsPtr def_levels_data;
   size_t def_levels_length;
@@ -1460,7 +1460,7 @@ Status StructImpl::GetRepLevels(ValueLevelsPtr* data, size_t* length) {
 
 Status StructImpl::NextBatch(int batch_size, std::shared_ptr<Array>* out) {
   std::vector<std::shared_ptr<Array>> children_arrays;
-  std::shared_ptr<MutableBuffer> null_bitmap;
+  std::shared_ptr<Buffer> null_bitmap;
   int64_t null_count;
 
   // Gather children arrays and def levels

src/parquet/statistics-test.cc

@@ -194,8 +194,9 @@ bool* TestRowGroupStatistics<BooleanType>::GetValuesPointer(std::vector<bool>& v
 }
 
 template <typename TestType>
-typename std::vector<typename TestType::c_type> TestRowGroupStatistics<
-    TestType>::GetDeepCopy(const std::vector<typename TestType::c_type>& values) {
+typename std::vector<typename TestType::c_type>
+TestRowGroupStatistics<TestType>::GetDeepCopy(
+    const std::vector<typename TestType::c_type>& values) {
   return values;
 }
 
@@ -478,7 +479,7 @@ class TestStatistics : public ::testing::Test {
 };
 
 using CompareTestTypes = ::testing::Types<Int32Type, Int64Type, Int96Type, FloatType,
-                                   DoubleType, ByteArrayType, FLBAType>;
+                                          DoubleType, ByteArrayType, FLBAType>;
 
 // TYPE::INT32
 template <>
@@ -680,8 +681,5 @@ TEST_F(TestStatisticsFLBA, UnknownSortOrder) {
   ASSERT_FALSE(cc_metadata->is_stats_set());
 }
 
-
-
-
 }  // namespace test
 }  // namespace parquet