ARROW-1808: [C++] Make RecordBatch, Table virtual interfaces for column access

.travis.yml

@@ -55,6 +55,7 @@ matrix:
     - export ARROW_TRAVIS_VALGRIND=1
     - export ARROW_TRAVIS_PLASMA=1
     - export ARROW_TRAVIS_CLANG_FORMAT=1
+    - export ARROW_BUILD_WARNING_LEVEL=CHECKIN
     - export CC="clang-4.0"
     - export CXX="clang++-4.0"
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
@@ -74,6 +75,7 @@ matrix:
     before_script:
     - export ARROW_TRAVIS_USE_TOOLCHAIN=1
     - export ARROW_TRAVIS_PLASMA=1
+    - export ARROW_BUILD_WARNING_LEVEL=CHECKIN
     - travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh

c_glib/arrow-glib/record-batch.cpp

@@ -150,9 +150,8 @@ garrow_record_batch_new(GArrowSchema *schema,
   }
 
   auto arrow_record_batch =
-    std::make_shared<arrow::RecordBatch>(garrow_schema_get_raw(schema),
-                                         n_rows,
-                                         arrow_columns);
+    arrow::RecordBatch::Make(garrow_schema_get_raw(schema),
+                             n_rows, arrow_columns);
   return garrow_record_batch_new_raw(&arrow_record_batch);
 }
 

c_glib/arrow-glib/table.cpp

@@ -143,8 +143,7 @@ garrow_table_new(GArrowSchema *schema,
   }
 
   auto arrow_table =
-    std::make_shared<arrow::Table>(garrow_schema_get_raw(schema),
-                                   arrow_columns);
+    arrow::Table::Make(garrow_schema_get_raw(schema), arrow_columns);
   return garrow_table_new_raw(&arrow_table);
 }
 

c_glib/test/test-file-writer.rb

@@ -19,14 +19,18 @@ class TestFileWriter < Test::Unit::TestCase
   include Helper::Buildable
 
   def test_write_record_batch
+    data = [true]
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    schema = Arrow::Schema.new([field])
+
     tempfile = Tempfile.open("arrow-ipc-file-writer")
     output = Arrow::FileOutputStream.new(tempfile.path, false)
     begin
-      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
-      schema = Arrow::Schema.new([field])
       file_writer = Arrow::RecordBatchFileWriter.new(output, schema)
       begin
-        record_batch = Arrow::RecordBatch.new(schema, 0, [])
+        record_batch = Arrow::RecordBatch.new(schema,
+                                              data.size,
+                                              [build_boolean_array(data)])
         file_writer.write_record_batch(record_batch)
       ensure
         file_writer.close
@@ -38,8 +42,12 @@ def test_write_record_batch
     input = Arrow::MemoryMappedInputStream.new(tempfile.path)
     begin
       file_reader = Arrow::RecordBatchFileReader.new(input)
-      assert_equal(["enabled"],
+      assert_equal([field.name],
                    file_reader.schema.fields.collect(&:name))
+      assert_equal(Arrow::RecordBatch.new(schema,
+                                          data.size,
+                                          [build_boolean_array(data)]),
+                   file_reader.read_record_batch(0))
     ensure
       input.close
     end

c_glib/test/test-gio-input-stream.rb

@@ -16,15 +16,21 @@
 # under the License.
 
 class TestGIOInputStream < Test::Unit::TestCase
+  include Helper::Buildable
+
   def test_reader_backend
+    data = [true]
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    schema = Arrow::Schema.new([field])
+
     tempfile = Tempfile.open("arrow-gio-input-stream")
     output = Arrow::FileOutputStream.new(tempfile.path, false)
     begin
-      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
-      schema = Arrow::Schema.new([field])
       file_writer = Arrow::RecordBatchFileWriter.new(output, schema)
       begin
-        record_batch = Arrow::RecordBatch.new(schema, 0, [])
+        record_batch = Arrow::RecordBatch.new(schema,
+                                              data.size,
+                                              [build_boolean_array(data)])
         file_writer.write_record_batch(record_batch)
       ensure
         file_writer.close
@@ -38,8 +44,12 @@ def test_reader_backend
     input = Arrow::GIOInputStream.new(input_stream)
     begin
       file_reader = Arrow::RecordBatchFileReader.new(input)
-      assert_equal(["enabled"],
+      assert_equal([field.name],
                    file_reader.schema.fields.collect(&:name))
+      assert_equal(Arrow::RecordBatch.new(schema,
+                                          data.size,
+                                          [build_boolean_array(data)]),
+                   file_reader.read_record_batch(0))
     ensure
       input.close
     end

c_glib/test/test-gio-output-stream.rb

@@ -16,17 +16,23 @@
 # under the License.
 
 class TestGIOOutputStream < Test::Unit::TestCase
+  include Helper::Buildable
+
   def test_writer_backend
+    data = [true]
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    schema = Arrow::Schema.new([field])
+
     tempfile = Tempfile.open("arrow-gio-output-stream")
     file = Gio::File.new_for_path(tempfile.path)
     output_stream = file.append_to(:none)
     output = Arrow::GIOOutputStream.new(output_stream)
     begin
-      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
-      schema = Arrow::Schema.new([field])
       file_writer = Arrow::RecordBatchFileWriter.new(output, schema)
       begin
-        record_batch = Arrow::RecordBatch.new(schema, 0, [])
+        record_batch = Arrow::RecordBatch.new(schema,
+                                              data.size,
+                                              [build_boolean_array(data)])
         file_writer.write_record_batch(record_batch)
       ensure
         file_writer.close
@@ -38,8 +44,12 @@ def test_writer_backend
     input = Arrow::MemoryMappedInputStream.new(tempfile.path)
     begin
       file_reader = Arrow::RecordBatchFileReader.new(input)
-      assert_equal(["enabled"],
+      assert_equal([field.name],
                    file_reader.schema.fields.collect(&:name))
+      assert_equal(Arrow::RecordBatch.new(schema,
+                                          data.size,
+                                          [build_boolean_array(data)]),
+                   file_reader.read_record_batch(0))
     ensure
       input.close
     end

c_glib/test/test-stream-writer.rb

@@ -19,17 +19,19 @@ class TestStreamWriter < Test::Unit::TestCase
   include Helper::Buildable
 
   def test_write_record_batch
+    data = [true]
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    schema = Arrow::Schema.new([field])
+
     tempfile = Tempfile.open("arrow-ipc-stream-writer")
     output = Arrow::FileOutputStream.new(tempfile.path, false)
     begin
-      field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
-      schema = Arrow::Schema.new([field])
       stream_writer = Arrow::RecordBatchStreamWriter.new(output, schema)
       begin
         columns = [
-          build_boolean_array([true]),
+          build_boolean_array(data),
         ]
-        record_batch = Arrow::RecordBatch.new(schema, 1, columns)
+        record_batch = Arrow::RecordBatch.new(schema, data.size, columns)
         stream_writer.write_record_batch(record_batch)
       ensure
         stream_writer.close
@@ -41,10 +43,12 @@ def test_write_record_batch
     input = Arrow::MemoryMappedInputStream.new(tempfile.path)
     begin
       stream_reader = Arrow::RecordBatchStreamReader.new(input)
-      assert_equal(["enabled"],
+      assert_equal([field.name],
                    stream_reader.schema.fields.collect(&:name))
-      assert_equal(true,
-                   stream_reader.read_next.get_column(0).get_value(0))
+      assert_equal(Arrow::RecordBatch.new(schema,
+                                          data.size,
+                                          [build_boolean_array(data)]),
+                   stream_reader.read_next)
       assert_nil(stream_reader.read_next)
     ensure
       input.close

ci/travis_before_script_cpp.sh

@@ -91,12 +91,14 @@ fi
 if [ $TRAVIS_OS_NAME == "linux" ]; then
     cmake $CMAKE_COMMON_FLAGS \
           $CMAKE_LINUX_FLAGS \
-          -DBUILD_WARNING_LEVEL=CHECKIN \
+          -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
+          -DBUILD_WARNING_LEVEL=$ARROW_BUILD_WARNING_LEVEL \
           $ARROW_CPP_DIR
 else
     cmake $CMAKE_COMMON_FLAGS \
           $CMAKE_OSX_FLAGS \
-          -DBUILD_WARNING_LEVEL=CHECKIN \
+          -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
+          -DBUILD_WARNING_LEVEL=$ARROW_BUILD_WARNING_LEVEL \
           $ARROW_CPP_DIR
 fi
 

ci/travis_env_common.sh

@@ -38,6 +38,9 @@ export ARROW_PYTHON_PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
 
 export CMAKE_EXPORT_COMPILE_COMMANDS=1
 
+export ARROW_BUILD_TYPE=${ARROW_BUILD_TYPE:=debug}
+export ARROW_BUILD_WARNING_LEVEL=${ARROW_BUILD_WARNING_LEVEL:=Production}
+
 if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
   # C++ toolchain
   export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain

ci/travis_script_python.sh

@@ -63,6 +63,7 @@ cmake -GNinja \
       -DARROW_BUILD_UTILITIES=off \
       -DARROW_PLASMA=on \
       -DARROW_PYTHON=on \
+      -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
       -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
       $ARROW_CPP_DIR
 
@@ -78,6 +79,8 @@ if [ "$PYTHON_VERSION" == "2.7" ]; then
   pip install futures
 fi
 
+export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE
+
 pip install -r requirements.txt
 python setup.py build_ext --with-parquet --with-plasma \
        install --single-version-externally-managed --record=record.text

cpp/src/arrow/CMakeLists.txt

@@ -22,6 +22,7 @@ set(ARROW_SRCS
   compare.cc
   memory_pool.cc
   pretty_print.cc
+  record_batch.cc
   status.cc
   table.cc
   table_builder.cc
@@ -144,6 +145,7 @@ install(FILES
   compare.h
   memory_pool.h
   pretty_print.h
+  record_batch.h
   status.h
   table.h
   table_builder.h

cpp/src/arrow/api.h

@@ -26,6 +26,7 @@
 #include "arrow/compare.h"
 #include "arrow/memory_pool.h"
 #include "arrow/pretty_print.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/table_builder.h"

cpp/src/arrow/array.h

@@ -279,6 +279,8 @@ class ARROW_EXPORT Array {
   ARROW_DISALLOW_COPY_AND_ASSIGN(Array);
 };
 
+using ArrayVector = std::vector<std::shared_ptr<Array>>;
+
 static inline std::ostream& operator<<(std::ostream& os, const Array& x) {
   os << x.ToString();
   return os;

cpp/src/arrow/builder.cc

@@ -28,7 +28,6 @@
 #include "arrow/buffer.h"
 #include "arrow/compare.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"

cpp/src/arrow/builder.h

@@ -29,7 +29,6 @@
 #include "arrow/buffer.h"
 #include "arrow/memory_pool.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"

cpp/src/arrow/column-benchmark.cc

@@ -19,6 +19,7 @@
 
 #include "arrow/array.h"
 #include "arrow/memory_pool.h"
+#include "arrow/table.h"
 #include "arrow/test-util.h"
 
 namespace arrow {

cpp/src/arrow/compute/kernel.h

@@ -22,6 +22,7 @@
 #include <vector>
 
 #include "arrow/array.h"
+#include "arrow/record_batch.h"
 #include "arrow/table.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/variant.h"

cpp/src/arrow/gpu/cuda_arrow_ipc.cc

@@ -27,8 +27,8 @@
 #include "arrow/ipc/message.h"
 #include "arrow/ipc/reader.h"
 #include "arrow/ipc/writer.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/util/visibility.h"
 
 #include "arrow/gpu/cuda_context.h"

cpp/src/arrow/ipc/feather-test.cc

@@ -29,6 +29,7 @@
 #include "arrow/ipc/feather.h"
 #include "arrow/ipc/test-common.h"
 #include "arrow/pretty_print.h"
+#include "arrow/table.h"
 #include "arrow/test-util.h"
 
 namespace arrow {
@@ -376,8 +377,8 @@ TEST_F(TestTableWriter, TimeTypes) {
         schema->field(i)->type(), values->length(), buffers, values->null_count(), 0));
   }
 
-  RecordBatch batch(schema, values->length(), std::move(arrays));
-  CheckBatch(batch);
+  auto batch = RecordBatch::Make(schema, values->length(), std::move(arrays));
+  CheckBatch(*batch);
 }
 
 TEST_F(TestTableWriter, VLenPrimitiveRoundTrip) {

cpp/src/arrow/ipc/feather.cc

@@ -32,6 +32,7 @@
 #include "arrow/ipc/feather-internal.h"
 #include "arrow/ipc/feather_generated.h"
 #include "arrow/ipc/util.h"  // IWYU pragma: keep
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
 #include "arrow/table.h"
 #include "arrow/type.h"

cpp/src/arrow/ipc/ipc-json-test.cc

@@ -31,8 +31,8 @@
 #include "arrow/ipc/json.h"
 #include "arrow/ipc/test-common.h"
 #include "arrow/memory_pool.h"
+#include "arrow/record_batch.h"
 #include "arrow/status.h"
-#include "arrow/table.h"
 #include "arrow/test-util.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
@@ -269,7 +269,7 @@ TEST(TestJsonFileReadWrite, BasicRoundTrip) {
     std::vector<std::shared_ptr<Array>> arrays;
 
     MakeBatchArrays(schema, num_rows, &arrays);
-    auto batch = std::make_shared<RecordBatch>(schema, num_rows, arrays);
+    auto batch = RecordBatch::Make(schema, num_rows, arrays);
     batches.push_back(batch);
     ASSERT_OK(writer->WriteRecordBatch(*batch));
   }

cpp/src/arrow/ipc/ipc-read-write-benchmark.cc

@@ -63,7 +63,7 @@ std::shared_ptr<RecordBatch> MakeRecordBatch(int64_t total_size, int64_t num_fie
   }
 
   auto schema = std::make_shared<Schema>(fields);
-  return std::make_shared<RecordBatch>(schema, length, arrays);
+  return RecordBatch::Make(schema, length, arrays);
 }
 
 static void BM_WriteRecordBatch(benchmark::State& state) {  // NOLINT non-const reference