Proof of concept passing data around between Python and R

r/DESCRIPTION

@@ -44,6 +44,7 @@ Suggests:
     hms,
     knitr,
     lubridate,
+    reticulate,
     rmarkdown,
     testthat,
     tibble
@@ -75,6 +76,7 @@ Collate:
     'memory-pool.R'
     'message.R'
     'parquet.R'
+    'py-to-r.R'
     'read-record-batch.R'
     'read-table.R'
     'record-batch-reader.R'

r/R/arrow-package.R

@@ -39,6 +39,10 @@
       s3_register(m, cl)
     }
   }
+  s3_register("reticulate::py_to_r", "pyarrow.lib.Array")
+  s3_register("reticulate::py_to_r", "pyarrow.lib.RecordBatch")
+  s3_register("reticulate::r_to_py", "Array")
+  s3_register("reticulate::r_to_py", "RecordBatch")
   invisible()
 }
 

r/R/py-to-r.R

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+py_to_r.pyarrow.lib.Array <- function(x, ...) {
+  ptr <- allocate_arrow_array()
+  on.exit(delete_arrow_array(ptr))
+  x$`_export_to_c`(ptr)
+  Array$create(ImportArray(ptr))
+}
+
+r_to_py.Array <- function(x, convert = FALSE) {
+  ptr <- allocate_arrow_array()
+  on.exit(delete_arrow_array(ptr))
+  ExportArray(x, ptr)
+  pa <- reticulate::import("pyarrow", convert = convert)
+  pa$Array$`_import_from_c`(ptr)
+}
+
+py_to_r.pyarrow.lib.RecordBatch <- function(x, ...) {
+  ptr <- allocate_arrow_array()
+  on.exit(delete_arrow_array(ptr))
+  x$`_export_to_c`(ptr)
+  shared_ptr(RecordBatch, ImportRecordBatch(ptr))
+}
+
+r_to_py.RecordBatch <- function(x, convert = FALSE) {
+  ptr <- allocate_arrow_array()
+  on.exit(delete_arrow_array(ptr))
+  ExportRecordBatch(x, ptr)
+  pa <- reticulate::import("pyarrow", convert = convert)
+  pa$RecordBatch$`_import_from_c`(ptr)
+}

r/src/arrow_types.h

@@ -206,6 +206,7 @@ inline std::shared_ptr<T> extract(SEXP x) {
 
 #if defined(ARROW_R_WITH_ARROW)
 #include <arrow/api.h>
+#include <arrow/c/bridge.h>
 #include <arrow/compute/api.h>
 #include <arrow/csv/reader.h>
 #include <arrow/dataset/api.h>

r/src/py-to-r.cpp

@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "./arrow_types.h"
+
+#if defined(ARROW_R_WITH_ARROW)
+
+// [[arrow::export]]
+std::shared_ptr<arrow::Array> ImportArray(size_t array) {
+  std::shared_ptr<arrow::Array> out;
+  STOP_IF_NOT_OK(arrow::ImportArray(reinterpret_cast<struct ArrowArray*>(array), &out));
+  return out;
+}
+
+// [[arrow::export]]
+std::shared_ptr<arrow::RecordBatch> ImportRecordBatch(size_t array) {
+  std::shared_ptr<arrow::RecordBatch> out;
+  STOP_IF_NOT_OK(
+      arrow::ImportRecordBatch(reinterpret_cast<struct ArrowArray*>(array), &out));
+  return out;
+}
+
+// [[arrow::export]]
+size_t allocate_arrow_array() { return reinterpret_cast<size_t>(new ArrowArray); }
+
+// [[arrow::export]]
+void delete_arrow_array(size_t ptr) { delete reinterpret_cast<struct ArrowArray*>(ptr); }
+
+// [[arrow::export]]
+void ExportArray(const std::shared_ptr<arrow::Array>& array, size_t ptr) {
+  STOP_IF_NOT_OK(arrow::ExportArray(*array, reinterpret_cast<struct ArrowArray*>(ptr)));
+}
+
+// [[arrow::export]]
+void ExportRecordBatch(const std::shared_ptr<arrow::RecordBatch>& array, size_t ptr) {
+  STOP_IF_NOT_OK(
+      arrow::ExportRecordBatch(*array, reinterpret_cast<struct ArrowArray*>(ptr)));
+}
+
+#endif

r/tests/testthat/helper-skip.R

@@ -21,3 +21,10 @@ skip_if_not_available <- function(feature) {
     skip(paste("Arrow C++ not built with support for", feature))
   }
 }
+
+skip_if_no_pyarrow <- function() {
+  skip_if_not_installed("reticulate")
+  if (!reticulate::py_module_available("pyarrow")) {
+    skip("pyarrow not available for testing")
+  }
+}

r/tests/testthat/test-python.R

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+context("To/from Python")
+
+test_that("Array from Python", {
+  skip_if_no_pyarrow()
+  pa <- reticulate::import("pyarrow")
+  py <- pa$array(c(1, 2, 3))
+  expect_equal(py, Array$create(c(1, 2, 3)))
+})
+
+test_that("Array to Python", {
+  skip_if_no_pyarrow()
+  pa <- reticulate::import("pyarrow", convert=FALSE)
+  r <- Array$create(c(1, 2, 3))
+  py <- pa$concat_arrays(list(r))
+  expect_is(py, "pyarrow.lib.Array")
+  expect_equal(reticulate::py_to_r(py), r)
+})
+
+test_that("RecordBatch to/from Python", {
+  skip_if_no_pyarrow()
+  pa <- reticulate::import("pyarrow", convert=FALSE)
+  batch <- record_batch(col1=c(1, 2, 3), col2=letters[1:3])
+  py <- reticulate::r_to_py(batch)
+  expect_is(py, "pyarrow.lib.RecordBatch")
+  expect_equal(reticulate::py_to_r(py), batch)
+})