ARROW-7808: [Java][Dataset] Implement Dataset Java API by JNI to C++

.github/workflows/java_jni.yml

@@ -45,7 +45,7 @@ env:
 jobs:
 
   docker:
-    name: AMD64 Debian 9 Java JNI (Gandiva, Plasma, ORC)
+    name: AMD64 Debian 9 Java JNI (Gandiva, Plasma, ORC, Dataset)
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     strategy:

ci/docker/linux-apt-jni.dockerfile

@@ -68,13 +68,14 @@ RUN wget -nv -O - https://github.com/Kitware/CMake/releases/download/v${cmake}/c
 ENV PATH=/opt/cmake-${cmake}-Linux-x86_64/bin:$PATH
 
 ENV ARROW_BUILD_TESTS=OFF \
+    ARROW_DATASET=ON \
     ARROW_FLIGHT=OFF \
     ARROW_GANDIVA_JAVA=ON \
     ARROW_GANDIVA=ON \
     ARROW_HOME=/usr/local \
     ARROW_JNI=ON \
     ARROW_ORC=ON \
-    ARROW_PARQUET=OFF \
+    ARROW_PARQUET=ON \
     ARROW_PLASMA_JAVA_CLIENT=ON \
     ARROW_PLASMA=ON \
     ARROW_USE_CCACHE=ON \

ci/scripts/java_test.sh

@@ -34,8 +34,8 @@ pushd ${source_dir}
 
 ${mvn} test
 
-if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then
-  ${mvn} test -Parrow-jni -pl adapter/orc,gandiva -Darrow.cpp.build.dir=${cpp_build_dir}
+if [ "${ARROW_JNI}" = "ON" ]; then
+  ${mvn} test -Parrow-jni -pl adapter/orc,gandiva,dataset -Darrow.cpp.build.dir=${cpp_build_dir}
 fi
 
 if [ "${ARROW_PLASMA}" = "ON" ]; then

cpp/.gitignore

@@ -25,6 +25,9 @@ build/
 *-build/
 Testing/
 
+# Build directories created by Clion
+cmake-build-*/
+
 #########################################
 # Editor temporary/working/backup files #
 .#*

cpp/src/arrow/dataset/discovery.cc

@@ -208,6 +208,17 @@ Result<std::shared_ptr<DatasetFactory>> FileSystemDatasetFactory::Make(
               std::move(options));
 }
 
+Result<std::shared_ptr<DatasetFactory>> FileSystemDatasetFactory::Make(
+    std::string uri, std::shared_ptr<FileFormat> format,
+    FileSystemFactoryOptions options) {
+  std::string internal_path;
+  ARROW_ASSIGN_OR_RAISE(std::shared_ptr<fs::FileSystem> filesystem,
+                        arrow::fs::FileSystemFromUri(uri, &internal_path))
+  ARROW_ASSIGN_OR_RAISE(fs::FileInfo file_info, filesystem->GetFileInfo(internal_path))
+  return std::shared_ptr<DatasetFactory>(new FileSystemDatasetFactory(
+      {file_info}, std::move(filesystem), std::move(format), std::move(options)));
+}
+
 Result<std::vector<std::shared_ptr<Schema>>> FileSystemDatasetFactory::InspectSchemas(
     InspectOptions options) {
   std::vector<std::shared_ptr<Schema>> schemas;

cpp/src/arrow/dataset/discovery.h

@@ -216,6 +216,16 @@ class ARROW_DS_EXPORT FileSystemDatasetFactory : public DatasetFactory {
       std::shared_ptr<fs::FileSystem> filesystem, fs::FileSelector selector,
       std::shared_ptr<FileFormat> format, FileSystemFactoryOptions options);
 
+  /// \brief Build a FileSystemDatasetFactory from an uri including filesystem
+  /// information.
+  ///
+  /// \param[in] uri passed to FileSystemDataset
+  /// \param[in] format passed to FileSystemDataset
+  /// \param[in] options see FileSystemFactoryOptions for more information.
+  static Result<std::shared_ptr<DatasetFactory>> Make(std::string uri,
+                                                      std::shared_ptr<FileFormat> format,
+                                                      FileSystemFactoryOptions options);
+
   Result<std::vector<std::shared_ptr<Schema>>> InspectSchemas(
       InspectOptions options) override;
 

cpp/src/jni/CMakeLists.txt

@@ -18,7 +18,10 @@
 #
 # arrow_jni
 #
-
 if(ARROW_ORC)
   add_subdirectory(orc)
 endif()
+
+if(ARROW_DATASET)
+  add_subdirectory(dataset)
+endif()

cpp/src/jni/dataset/CMakeLists.txt

@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitationsn
+# under the License.
+
+#
+# arrow_dataset_jni
+#
+
+project(arrow_dataset_jni)
+
+cmake_minimum_required(VERSION 3.11)
+
+find_package(JNI REQUIRED)
+
+add_custom_target(arrow_dataset_jni)
+
+set(JNI_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated")
+
+add_subdirectory(../../../../java/dataset ./java)
+
+set(ARROW_BUILD_STATIC OFF)
+
+set(ARROW_DATASET_JNI_LIBS arrow_dataset_static)
+
+set(ARROW_DATASET_JNI_SOURCES jni_wrapper.cc jni_util.cc)
+
+add_arrow_lib(arrow_dataset_jni
+              BUILD_SHARED
+              SOURCES
+              ${ARROW_DATASET_JNI_SOURCES}
+              OUTPUTS
+              ARROW_DATASET_JNI_LIBRARIES
+              SHARED_PRIVATE_LINK_LIBS
+              ${ARROW_DATASET_JNI_LIBS}
+              STATIC_LINK_LIBS
+              ${ARROW_DATASET_JNI_LIBS}
+              EXTRA_INCLUDES
+              ${JNI_HEADERS_DIR}
+              PRIVATE_INCLUDES
+              ${JNI_INCLUDE_DIRS}
+              DEPENDENCIES
+              arrow_static
+              arrow_dataset_java)
+
+add_dependencies(arrow_dataset_jni ${ARROW_DATASET_JNI_LIBRARIES})
+
+add_arrow_test(dataset_jni_test
+               SOURCES
+               jni_util_test.cc
+               jni_util.cc
+               EXTRA_INCLUDES
+               ${JNI_INCLUDE_DIRS})