ARROW-18042: [Java] Distribute Apple M1 compatible JNI libraries via mavencentral

.github/workflows/java_jni.yml

@@ -41,7 +41,7 @@ concurrency:
 
 permissions:
   contents: read
-  
+
 env:
   DOCKER_VOLUME_PREFIX: ".docker/"
 

ci/scripts/java_build.sh

@@ -93,7 +93,7 @@ fi
 
 if [ "${ARROW_PLASMA}" = "ON" ]; then
   pushd ${source_dir}/plasma
-  ${mvn} clean install
+  ${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} clean install
   popd
 fi
 

ci/scripts/java_jni_macos_build.sh

@@ -21,8 +21,17 @@ set -ex
 
 arrow_dir=${1}
 build_dir=${2}
+normalized_arch=$(arch)
+case ${normalized_arch} in
+  arm64)
+    normalized_arch=aarch_64
+    ;;
+  i386)
+    normalized_arch=x86_64
+    ;;
+esac
 # The directory where the final binaries will be stored when scripts finish
-dist_dir=${3}
+dist_dir=${3}/${normalized_arch}
 
 echo "=== Clear output directories and leftovers ==="
 # Clear output directories and leftovers

ci/scripts/java_jni_manylinux_build.sh

@@ -22,7 +22,7 @@ set -ex
 arrow_dir=${1}
 build_dir=${2}
 # The directory where the final binaries will be stored when scripts finish
-dist_dir=${3}
+dist_dir=${3}/$(arch)
 
 echo "=== Clear output directories and leftovers ==="
 # Clear output directories and leftovers

ci/scripts/java_jni_windows_build.sh

@@ -22,7 +22,7 @@ set -ex
 arrow_dir=${1}
 build_dir=${2}
 # The directory where the final binaries will be stored when scripts finish
-dist_dir=${3}
+dist_dir=${3}/x86_64
 
 echo "=== Clear output directories and leftovers ==="
 # Clear output directories and leftovers

ci/scripts/java_test.sh

@@ -60,7 +60,7 @@ if [ "${#projects[@]}" -gt 0 ]; then
   if [ "${ARROW_PLASMA}" = "ON" ]; then
     pushd ${source_dir}/plasma
     java -cp target/test-classes:target/classes \
-         -Djava.library.path=${java_jni_dist_dir} \
+         -Djava.library.path=${java_jni_dist_dir}/$(arch) \
          org.apache.arrow.plasma.PlasmaClientTest
     popd
   fi

cpp/Brewfile

@@ -31,7 +31,6 @@ brew "grpc"
 brew "llvm@14"
 brew "lz4"
 brew "ninja"
-brew "numpy"
 brew "[email protected]"
 brew "protobuf"
 brew "python"

dev/tasks/java-jars/github.yml

@@ -28,6 +28,8 @@ jobs:
       {{ macros.github_checkout_arrow()|indent }}
       {{ macros.github_install_archery()|indent }}
       - name: Build C++ libraries
+        env:
+        {{ macros.github_set_sccache_envvars()|indent(8) }}
         run: |
           archery docker run \
             -e ARROW_JAVA_BUILD=OFF \
@@ -48,47 +50,65 @@ jobs:
     {% endif %}
 
   build-cpp-macos:
-    name: Build C++ libraries macOS
-    runs-on: macos-latest
+    {% set arch = '${{ matrix.platform.arch }}' %}
+    name: Build C++ libraries macOS {{ arch }}
+    runs-on: {{ '${{ matrix.platform.runs_on }}' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - { runs_on: ["macos-latest"], arch: "x86_64"}
+          - { runs_on: ["self-hosted", "macOS", "arm64", "devops-managed"], arch: "aarch_64" }
     env:
       MACOSX_DEPLOYMENT_TARGET: "10.13"
     steps:
       {{ macros.github_checkout_arrow()|indent }}
-      {{ macros.github_install_archery()|indent }}
+      - name: Set up Python
+        if: |
+          !contains(matrix.platform.runs_on, 'self-hosted')
+        uses: actions/setup-python@v4
+        with:
+          cache: 'pip'
+          python-version: 3.8
+      - name: Install Archery
+        shell: bash
+        run: pip install -e arrow/dev/archery[all]
       - name: Install dependencies
         run: |
-          brew install --overwrite git
           brew bundle --file=arrow/cpp/Brewfile
           # We want to link aws-sdk-cpp statically but Homebrew's
           # aws-sdk-cpp provides only shared library. If we have
           # Homebrew's aws-sdk-cpp, our build mix Homebrew's
           # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
           # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
           brew uninstall aws-sdk-cpp
-      - name: Setup ccache
-        run: |
-          arrow/ci/scripts/ccache_setup.sh
+          brew bundle --file=arrow/java/Brewfile
       - name: Build C++ libraries
+        env:
+        {{ macros.github_set_sccache_envvars()|indent(8) }}
         run: |
           set -e
+          # make brew Java available to CMake
+          if [ "{{ arch }}" = "aarch_64" ]; then
+            export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
+          fi
           arrow/ci/scripts/java_jni_macos_build.sh \
             $GITHUB_WORKSPACE/arrow \
             $GITHUB_WORKSPACE/arrow/cpp-build \
             $GITHUB_WORKSPACE/arrow/java-dist
       - name: Compress into single artifact to keep directory structure
-        run: tar -cvzf arrow-shared-libs-macos.tar.gz arrow/java-dist/
+        run: tar -cvzf arrow-shared-libs-macos-{{ arch }}.tar.gz arrow/java-dist/
       - name: Upload artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: macos-shared-lib
-          path: arrow-shared-libs-macos.tar.gz
+          name: macos-shared-lib-{{ arch }}
+          path: arrow-shared-libs-macos-{{ arch }}.tar.gz
 
   build-cpp-windows:
     name: Build C++ libraries Windows
     runs-on: windows-2019
     steps:
       {{ macros.github_checkout_arrow()|indent }}
-      {{ macros.github_install_archery()|indent }}
       - name: Set up Java
         uses: actions/setup-java@v3
         with:
@@ -97,8 +117,13 @@ jobs:
       - name: Download Timezone Database
         shell: bash
         run: arrow/ci/scripts/download_tz_database.sh
+      - name: Install sccache
+        shell: bash
+        run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache
       - name: Build C++ libraries
         shell: cmd
+        env:
+        {{ macros.github_set_sccache_envvars()|indent(8) }}
         run: |
           call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
           REM For ORC
@@ -122,39 +147,42 @@ jobs:
       - build-cpp-windows
     steps:
       {{ macros.github_checkout_arrow(fetch_depth=0)|indent }}
-      - name: Download Linux C++ Libraries
-        uses: actions/download-artifact@v2
+      - name: Download Libraries
+        uses: actions/download-artifact@v3
         with:
-          name: ubuntu-shared-lib
-      - name: Download macOS C++ libraries
-        uses: actions/download-artifact@v2
-        with:
-          name: macos-shared-lib
-      - name: Download Windows C++ libraries
-        uses: actions/download-artifact@v2
-        with:
-          name: windows-shared-lib
-      - name: Descompress artifacts
+          path: artifacts
+      - name: Decompress artifacts
         run: |
+          mv artifacts/*/*.tar.gz .
           tar -xvzf arrow-shared-libs-linux.tar.gz
-          tar -xvzf arrow-shared-libs-macos.tar.gz
+          tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz
+          tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz
           tar -xvzf arrow-shared-libs-windows.tar.gz
       - name: Test that shared libraries exist
         run: |
           set -x
-          test -f arrow/java-dist/libarrow_cdata_jni.dylib
-          test -f arrow/java-dist/libarrow_cdata_jni.so
-          test -f arrow/java-dist/arrow_cdata_jni.dll
-          test -f arrow/java-dist/libarrow_dataset_jni.dylib
-          test -f arrow/java-dist/libarrow_dataset_jni.so
-          test -f arrow/java-dist/arrow_dataset_jni.dll
-          test -f arrow/java-dist/libarrow_orc_jni.dylib
-          test -f arrow/java-dist/libarrow_orc_jni.so
-          test -f arrow/java-dist/arrow_orc_jni.dll
-          test -f arrow/java-dist/libgandiva_jni.dylib
-          test -f arrow/java-dist/libgandiva_jni.so
-          test -f arrow/java-dist/libplasma_java.dylib
-          test -f arrow/java-dist/libplasma_java.so
+
+          test -f arrow/java-dist/x86_64/libarrow_cdata_jni.so
+          test -f arrow/java-dist/x86_64/libarrow_dataset_jni.so
+          test -f arrow/java-dist/x86_64/libarrow_orc_jni.so
+          test -f arrow/java-dist/x86_64/libgandiva_jni.so
+          test -f arrow/java-dist/x86_64/libplasma_java.so
+
+          test -f arrow/java-dist/x86_64/libarrow_cdata_jni.dylib
+          test -f arrow/java-dist/x86_64/libarrow_dataset_jni.dylib
+          test -f arrow/java-dist/x86_64/libarrow_orc_jni.dylib
+          test -f arrow/java-dist/x86_64/libgandiva_jni.dylib
+          test -f arrow/java-dist/x86_64/libplasma_java.dylib
+
+          test -f arrow/java-dist/aarch_64/libarrow_cdata_jni.dylib
+          test -f arrow/java-dist/aarch_64/libarrow_dataset_jni.dylib
+          test -f arrow/java-dist/aarch_64/libarrow_orc_jni.dylib
+          test -f arrow/java-dist/aarch_64/libgandiva_jni.dylib
+          test -f arrow/java-dist/aarch_64/libplasma_java.dylib
+
+          test -f arrow/java-dist/x86_64/arrow_cdata_jni.dll
+          test -f arrow/java-dist/x86_64/arrow_dataset_jni.dll
+          test -f arrow/java-dist/x86_64/arrow_orc_jni.dll
       - name: Build bundled jar
         run: |
           set -e

docker-compose.yml

@@ -1210,7 +1210,7 @@ services:
     command:
       [ "/arrow/ci/scripts/cpp_build.sh /arrow /build &&
           /arrow/ci/scripts/python_build.sh /arrow /build &&
-          /arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java &&
+          /arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java/$$(arch) &&
           /arrow/ci/scripts/java_build.sh /arrow /build /tmp/dist/java &&
           /arrow/ci/scripts/java_cdata_integration.sh /arrow /tmp/dist/java" ]
 

docs/source/developers/java/building.rst

@@ -112,7 +112,7 @@ Maven
         $ export JAVA_HOME=<absolute path to your java home>
         $ java --version
         $ mvn generate-resources -Pgenerate-libs-cdata-all-os -N
-        $ ls -latr ../java-dist/lib
+        $ ls -latr ../java-dist/lib/<your system's architecture>
         |__ libarrow_cdata_jni.dylib
         |__ libarrow_cdata_jni.so
 
@@ -122,7 +122,7 @@ Maven
 
         $ cd arrow/java
         $ mvn generate-resources -Pgenerate-libs-cdata-all-os -N
-        $ dir "../java-dist/bin"
+        $ dir "../java-dist/bin/x86_64"
         |__ arrow_cdata_jni.dll
 
 - To build all JNI libraries (MacOS / Linux) except the JNI C Data Interface library:
@@ -133,7 +133,7 @@ Maven
         $ export JAVA_HOME=<absolute path to your java home>
         $ java --version
         $ mvn generate-resources -Pgenerate-libs-jni-macos-linux -N
-        $ ls -latr java-dist/lib/*_{jni,java}.*
+        $ ls -latr java-dist/lib/<your system's architecture>/*_{jni,java}.*
         |__ libarrow_dataset_jni.dylib
         |__ libarrow_orc_jni.dylib
         |__ libgandiva_jni.dylib
@@ -145,7 +145,7 @@ Maven
 
         $ cd arrow/java
         $ mvn generate-resources -Pgenerate-libs-jni-windows -N
-        $ dir "../java-dist/bin"
+        $ dir "../java-dist/bin/x86_64"
         |__ arrow_dataset_jni.dll
 
 CMake
@@ -164,7 +164,7 @@ CMake
             -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF \
             -DBUILD_TESTING=OFF \
             -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_INSTALL_LIBDIR=lib \
+            -DCMAKE_INSTALL_LIBDIR=lib/<your system's architecture> \
             -DCMAKE_INSTALL_PREFIX=java-dist
         $ cmake --build java-cdata --target install --config Release
         $ ls -latr java-dist/lib
@@ -184,7 +184,7 @@ CMake
             -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF ^
             -DBUILD_TESTING=OFF ^
             -DCMAKE_BUILD_TYPE=Release ^
-            -DCMAKE_INSTALL_LIBDIR=lib ^
+            -DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^
             -DCMAKE_INSTALL_PREFIX=java-dist
         $ cmake --build java-cdata --target install --config Release
         $ dir "java-dist/bin"
@@ -219,7 +219,7 @@ CMake
             -DARROW_S3=ON \
             -DARROW_USE_CCACHE=ON \
             -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_INSTALL_LIBDIR=lib \
+            -DCMAKE_INSTALL_LIBDIR=lib/<your system's architecture> \
             -DCMAKE_INSTALL_PREFIX=java-dist \
             -DCMAKE_UNITY_BUILD=ON
         $ cmake --build cpp-jni --target install --config Release
@@ -230,11 +230,11 @@ CMake
             -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON \
             -DBUILD_TESTING=OFF \
             -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_INSTALL_LIBDIR=lib \
+            -DCMAKE_INSTALL_LIBDIR=lib/<your system's architecture> \
             -DCMAKE_INSTALL_PREFIX=java-dist \
             -DCMAKE_PREFIX_PATH=$PWD/java-dist
         $ cmake --build java-jni --target install --config Release
-        $ ls -latr java-dist/lib/*_{jni,java}.*
+        $ ls -latr java-dist/lib/<your system's architecture>/*_{jni,java}.*
         |__ libarrow_dataset_jni.dylib
         |__ libarrow_orc_jni.dylib
         |__ libgandiva_jni.dylib
@@ -265,7 +265,7 @@ CMake
             -DARROW_WITH_ZLIB=ON ^
             -DARROW_WITH_ZSTD=ON ^
             -DCMAKE_BUILD_TYPE=Release ^
-            -DCMAKE_INSTALL_LIBDIR=lib ^
+            -DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^
             -DCMAKE_INSTALL_PREFIX=java-dist ^
             -DCMAKE_UNITY_BUILD=ON ^
             -GNinja
@@ -282,7 +282,7 @@ CMake
             -DARROW_JAVA_JNI_ENABLE_PLASMA=OFF ^
             -DBUILD_TESTING=OFF ^
             -DCMAKE_BUILD_TYPE=Release ^
-            -DCMAKE_INSTALL_LIBDIR=lib ^
+            -DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^
             -DCMAKE_INSTALL_PREFIX=java-dist ^
             -DCMAKE_PREFIX_PATH=$PWD/java-dist
         $ cmake --build java-jni --target install --config Release
@@ -296,7 +296,7 @@ Archery
 
     $ cd arrow
     $ archery docker run java-jni-manylinux-2014
-    $ ls -latr java-dist/
+    $ ls -latr java-dist/<your system's architecture>/
     |__ libarrow_cdata_jni.so
     |__ libarrow_dataset_jni.so
     |__ libarrow_orc_jni.so

java/Brewfile

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+brew "openjdk@11"
+brew "sccache"