SKYHOOK-202: [Java] Add Java/JNI Dataset API Bindings for Rados Parquet

uccross · Jul 25, 2021 · 07b2f20 · 07b2f20
1 parent 64181c0
commit 07b2f20
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 5 deletions.
diff --git a/ci/docker/linux-apt-jni.dockerfile b/ci/docker/linux-apt-jni.dockerfile
@@ -51,6 +51,9 @@ RUN apt-get update -y -q && \
         liblz4-dev \
         libre2-dev \
         libsnappy-dev \
+        libradospp-dev \
+        rados-objclass-dev \
+        python3-rados \
         libssl-dev \
         llvm-${llvm}-dev \
         make \

diff --git a/cpp/src/arrow/adapters/arrow-rados-cls/scripts/deploy_skyhook.sh b/cpp/src/arrow/adapters/arrow-rados-cls/scripts/deploy_skyhook.sh
@@ -41,14 +41,18 @@ apt install -y python3 \
                cmake \
                libradospp-dev \
                rados-objclass-dev \
-               llvm
+               llvm \
+               default-jdk \
+               maven
 
 if [ ! -d "/tmp/arrow" ]; then
   git clone https://github.com/uccross/arrow /tmp/arrow
+  git submodule update --init --recursive
 fi
 
 cd /tmp/arrow
-git submodule update --init --recursive
+git fetch origin $BRANCH
+git pull
 git checkout $BRANCH
 mkdir -p cpp/release
 cd cpp/release
@@ -103,10 +107,14 @@ if [[ "${DEPLOY_CLS_LIBS}" == "true" ]]; then
 fi
 
 if [[ "${BUILD_JAVA_BINDINGS}" == "true" ]]; then
-    apt install -y default-jdk maven
+    mkdir -p /tmp/arrow/java/dist
+    cp -r /tmp/arrow/cpp/release/release/libarrow_dataset_jni.so* /tmp/arrow/java/dist
+
+    mvn="mvn -B -DskipTests -Dcheckstyle.skip -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
+    mvn="${mvn} -T 2C"
+
     cd /tmp/arrow/java
-    mvn clean install
-    mvn clean install -P arrow-jni -pl format,memory,vector -am -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release -Dmaven.test.skip=true -Dcheckstyle.skip -Dos.detected.name=linux -Dos.detected.arch=x86_64 -Dos.detected.classifier=linux-x86_64
+    ${mvn} clean install package -P arrow-jni -pl dataset,format,memory,vector -am -Darrow.cpp.build.dir=/tmp/arrow/cpp/release/release
 fi
 
 export LD_LIBRARY_PATH=/usr/local/lib

diff --git a/cpp/src/jni/dataset/jni_wrapper.cc b/cpp/src/jni/dataset/jni_wrapper.cc
@@ -20,6 +20,7 @@
 #include "arrow/array.h"
 #include "arrow/dataset/api.h"
 #include "arrow/dataset/file_base.h"
+#include "arrow/dataset/file_rados_parquet.h"
 #include "arrow/filesystem/localfs.h"
 #include "arrow/ipc/api.h"
 #include "arrow/util/iterator.h"
@@ -81,6 +82,9 @@ arrow::Result<std::shared_ptr<arrow::dataset::FileFormat>> GetFileFormat(
   switch (file_format_id) {
     case 0:
       return std::make_shared<arrow::dataset::ParquetFileFormat>();
+    case 1:
+      return std::make_shared<arrow::dataset::RadosParquetFileFormat>(
+        "/etc/ceph/ceph.conf", "cephfs_data", "client.admin", "ceph", "arrow");
     default:
       std::string error_message =
           "illegal file format id: " + std::to_string(file_format_id);

diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/file/FileFormat.java b/java/dataset/src/main/java/org/apache/arrow/dataset/file/FileFormat.java
@@ -21,6 +21,7 @@
  * File format definitions.
  */
 public enum FileFormat {
+  RADOS_PARQUET(1),
   PARQUET(0),
   NONE(-1);