diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml
index e78a7605eb0..ee80c2c9d96 100644
--- a/.github/workflows/java_jni.yml
+++ b/.github/workflows/java_jni.yml
@@ -41,7 +41,7 @@ concurrency:
 
 permissions:
   contents: read
-  
+
 env:
   DOCKER_VOLUME_PREFIX: ".docker/"
 
diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh
index a126e13bb05..3ea6fe28d63 100755
--- a/ci/scripts/java_build.sh
+++ b/ci/scripts/java_build.sh
@@ -93,7 +93,7 @@ fi
 
 if [ "${ARROW_PLASMA}" = "ON" ]; then
   pushd ${source_dir}/plasma
-  ${mvn} clean install
+  ${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} clean install
   popd
 fi
 
diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh
index 05d68f33597..912638b5082 100755
--- a/ci/scripts/java_jni_macos_build.sh
+++ b/ci/scripts/java_jni_macos_build.sh
@@ -21,8 +21,17 @@ set -ex
 
 arrow_dir=${1}
 build_dir=${2}
+normalized_arch=$(arch)
+case ${normalized_arch} in
+  arm64)
+    normalized_arch=aarch_64
+    ;;
+  i386)
+    normalized_arch=x86_64
+    ;;
+esac
 # The directory where the final binaries will be stored when scripts finish
-dist_dir=${3}
+dist_dir=${3}/${normalized_arch}
 
 echo "=== Clear output directories and leftovers ==="
 # Clear output directories and leftovers
diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh
index 884b76bc058..051ebe41d0d 100755
--- a/ci/scripts/java_jni_manylinux_build.sh
+++ b/ci/scripts/java_jni_manylinux_build.sh
@@ -22,7 +22,7 @@ set -ex
 arrow_dir=${1}
 build_dir=${2}
 # The directory where the final binaries will be stored when scripts finish
-dist_dir=${3}
+dist_dir=${3}/$(arch)
 
 echo "=== Clear output directories and leftovers ==="
 # Clear output directories and leftovers
diff --git a/ci/scripts/java_jni_windows_build.sh b/ci/scripts/java_jni_windows_build.sh
index 8d142bd4254..ce445db578f 100755
--- a/ci/scripts/java_jni_windows_build.sh
+++ b/ci/scripts/java_jni_windows_build.sh
@@ -22,7 +22,7 @@ set -ex
 arrow_dir=${1}
 build_dir=${2}
 # The directory where the final binaries will be stored when scripts finish
-dist_dir=${3}
+dist_dir=${3}/x86_64
 
 echo "=== Clear output directories and leftovers ==="
 # Clear output directories and leftovers
diff --git a/ci/scripts/java_test.sh b/ci/scripts/java_test.sh
index c062590e07a..46577b69625 100755
--- a/ci/scripts/java_test.sh
+++ b/ci/scripts/java_test.sh
@@ -60,7 +60,7 @@ if [ "${#projects[@]}" -gt 0 ]; then
   if [ "${ARROW_PLASMA}" = "ON" ]; then
     pushd ${source_dir}/plasma
     java -cp target/test-classes:target/classes \
-         -Djava.library.path=${java_jni_dist_dir} \
+         -Djava.library.path=${java_jni_dist_dir}/$(arch) \
          org.apache.arrow.plasma.PlasmaClientTest
     popd
   fi
diff --git a/cpp/Brewfile b/cpp/Brewfile
index 35941a92895..2ffc1232a09 100644
--- a/cpp/Brewfile
+++ b/cpp/Brewfile
@@ -31,7 +31,6 @@ brew "grpc"
 brew "llvm@14"
 brew "lz4"
 brew "ninja"
-brew "numpy"
 brew "openssl@1.1"
 brew "protobuf"
 brew "python"
diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml
index 763f5df5cd7..b3aa46bf717 100644
--- a/dev/tasks/java-jars/github.yml
+++ b/dev/tasks/java-jars/github.yml
@@ -28,6 +28,8 @@ jobs:
       {{ macros.github_checkout_arrow()|indent }}
       {{ macros.github_install_archery()|indent }}
       - name: Build C++ libraries
+        env:
+        {{ macros.github_set_sccache_envvars()|indent(8) }}
         run: |
           archery docker run \
             -e ARROW_JAVA_BUILD=OFF \
@@ -48,16 +50,31 @@ jobs:
     {% endif %}
 
   build-cpp-macos:
-    name: Build C++ libraries macOS
-    runs-on: macos-latest
+    {% set arch = '${{ matrix.platform.arch }}' %}
+    name: Build C++ libraries macOS {{ arch }}
+    runs-on: {{ '${{ matrix.platform.runs_on }}' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - { runs_on: ["macos-latest"], arch: "x86_64"}
+          - { runs_on: ["self-hosted", "macOS", "arm64", "devops-managed"], arch: "aarch_64" }
     env:
       MACOSX_DEPLOYMENT_TARGET: "10.13"
     steps:
       {{ macros.github_checkout_arrow()|indent }}
-      {{ macros.github_install_archery()|indent }}
+      - name: Set up Python
+        if: |
+          !contains(matrix.platform.runs_on, 'self-hosted')
+        uses: actions/setup-python@v4
+        with:
+          cache: 'pip'
+          python-version: 3.8
+      - name: Install Archery
+        shell: bash
+        run: pip install -e arrow/dev/archery[all]
       - name: Install dependencies
         run: |
-          brew install --overwrite git
           brew bundle --file=arrow/cpp/Brewfile
           # We want to link aws-sdk-cpp statically but Homebrew's
           # aws-sdk-cpp provides only shared library. If we have
@@ -65,30 +82,33 @@ jobs:
           # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
           # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
           brew uninstall aws-sdk-cpp
-      - name: Setup ccache
-        run: |
-          arrow/ci/scripts/ccache_setup.sh
+          brew bundle --file=arrow/java/Brewfile
       - name: Build C++ libraries
+        env:
+        {{ macros.github_set_sccache_envvars()|indent(8) }}
         run: |
           set -e
+          # make brew Java available to CMake
+          if [ "{{ arch }}" = "aarch_64" ]; then
+            export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
+          fi
           arrow/ci/scripts/java_jni_macos_build.sh \
             $GITHUB_WORKSPACE/arrow \
             $GITHUB_WORKSPACE/arrow/cpp-build \
             $GITHUB_WORKSPACE/arrow/java-dist
       - name: Compress into single artifact to keep directory structure
-        run: tar -cvzf arrow-shared-libs-macos.tar.gz arrow/java-dist/
+        run: tar -cvzf arrow-shared-libs-macos-{{ arch }}.tar.gz arrow/java-dist/
       - name: Upload artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: macos-shared-lib
-          path: arrow-shared-libs-macos.tar.gz
+          name: macos-shared-lib-{{ arch }}
+          path: arrow-shared-libs-macos-{{ arch }}.tar.gz
 
   build-cpp-windows:
     name: Build C++ libraries Windows
     runs-on: windows-2019
     steps:
       {{ macros.github_checkout_arrow()|indent }}
-      {{ macros.github_install_archery()|indent }}
       - name: Set up Java
         uses: actions/setup-java@v3
         with:
@@ -97,8 +117,13 @@ jobs:
       - name: Download Timezone Database
         shell: bash
         run: arrow/ci/scripts/download_tz_database.sh
+      - name: Install sccache
+        shell: bash
+        run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache
       - name: Build C++ libraries
         shell: cmd
+        env:
+        {{ macros.github_set_sccache_envvars()|indent(8) }}
         run: |
           call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
           REM For ORC
@@ -122,39 +147,42 @@ jobs:
       - build-cpp-windows
     steps:
       {{ macros.github_checkout_arrow(fetch_depth=0)|indent }}
-      - name: Download Linux C++ Libraries
-        uses: actions/download-artifact@v2
+      - name: Download Libraries
+        uses: actions/download-artifact@v3
         with:
-          name: ubuntu-shared-lib
-      - name: Download macOS C++ libraries
-        uses: actions/download-artifact@v2
-        with:
-          name: macos-shared-lib
-      - name: Download Windows C++ libraries
-        uses: actions/download-artifact@v2
-        with:
-          name: windows-shared-lib
-      - name: Descompress artifacts
+          path: artifacts
+      - name: Decompress artifacts
         run: |
+          mv artifacts/*/*.tar.gz .
           tar -xvzf arrow-shared-libs-linux.tar.gz
-          tar -xvzf arrow-shared-libs-macos.tar.gz
+          tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz
+          tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz
           tar -xvzf arrow-shared-libs-windows.tar.gz
       - name: Test that shared libraries exist
         run: |
           set -x
-          test -f arrow/java-dist/libarrow_cdata_jni.dylib
-          test -f arrow/java-dist/libarrow_cdata_jni.so
-          test -f arrow/java-dist/arrow_cdata_jni.dll
-          test -f arrow/java-dist/libarrow_dataset_jni.dylib
-          test -f arrow/java-dist/libarrow_dataset_jni.so
-          test -f arrow/java-dist/arrow_dataset_jni.dll
-          test -f arrow/java-dist/libarrow_orc_jni.dylib
-          test -f arrow/java-dist/libarrow_orc_jni.so
-          test -f arrow/java-dist/arrow_orc_jni.dll
-          test -f arrow/java-dist/libgandiva_jni.dylib
-          test -f arrow/java-dist/libgandiva_jni.so
-          test -f arrow/java-dist/libplasma_java.dylib
-          test -f arrow/java-dist/libplasma_java.so
+
+          test -f arrow/java-dist/x86_64/libarrow_cdata_jni.so
+          test -f arrow/java-dist/x86_64/libarrow_dataset_jni.so
+          test -f arrow/java-dist/x86_64/libarrow_orc_jni.so
+          test -f arrow/java-dist/x86_64/libgandiva_jni.so
+          test -f arrow/java-dist/x86_64/libplasma_java.so
+
+          test -f arrow/java-dist/x86_64/libarrow_cdata_jni.dylib
+          test -f arrow/java-dist/x86_64/libarrow_dataset_jni.dylib
+          test -f arrow/java-dist/x86_64/libarrow_orc_jni.dylib
+          test -f arrow/java-dist/x86_64/libgandiva_jni.dylib
+          test -f arrow/java-dist/x86_64/libplasma_java.dylib
+
+          test -f arrow/java-dist/aarch_64/libarrow_cdata_jni.dylib
+          test -f arrow/java-dist/aarch_64/libarrow_dataset_jni.dylib
+          test -f arrow/java-dist/aarch_64/libarrow_orc_jni.dylib
+          test -f arrow/java-dist/aarch_64/libgandiva_jni.dylib
+          test -f arrow/java-dist/aarch_64/libplasma_java.dylib
+
+          test -f arrow/java-dist/x86_64/arrow_cdata_jni.dll
+          test -f arrow/java-dist/x86_64/arrow_dataset_jni.dll
+          test -f arrow/java-dist/x86_64/arrow_orc_jni.dll
       - name: Build bundled jar
         run: |
           set -e
diff --git a/docker-compose.yml b/docker-compose.yml
index 66068a39d29..cd6c244ecfc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1210,7 +1210,7 @@ services:
     command:
       [ "/arrow/ci/scripts/cpp_build.sh /arrow /build &&
           /arrow/ci/scripts/python_build.sh /arrow /build &&
-          /arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java &&
+          /arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java/$$(arch) &&
           /arrow/ci/scripts/java_build.sh /arrow /build /tmp/dist/java &&
           /arrow/ci/scripts/java_cdata_integration.sh /arrow /tmp/dist/java" ]
 
diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst
index 00a9d3d1dff..54be993b370 100644
--- a/docs/source/developers/java/building.rst
+++ b/docs/source/developers/java/building.rst
@@ -112,7 +112,7 @@ Maven
         $ export JAVA_HOME=<absolute path to your java home>
         $ java --version
         $ mvn generate-resources -Pgenerate-libs-cdata-all-os -N
-        $ ls -latr ../java-dist/lib
+        $ ls -latr ../java-dist/lib/<your system's architecture>
         |__ libarrow_cdata_jni.dylib
         |__ libarrow_cdata_jni.so
 
@@ -122,7 +122,7 @@ Maven
 
         $ cd arrow/java
         $ mvn generate-resources -Pgenerate-libs-cdata-all-os -N
-        $ dir "../java-dist/bin"
+        $ dir "../java-dist/bin/x86_64"
         |__ arrow_cdata_jni.dll
 
 - To build all JNI libraries (MacOS / Linux) except the JNI C Data Interface library:
@@ -133,7 +133,7 @@ Maven
         $ export JAVA_HOME=<absolute path to your java home>
         $ java --version
         $ mvn generate-resources -Pgenerate-libs-jni-macos-linux -N
-        $ ls -latr java-dist/lib/*_{jni,java}.*
+        $ ls -latr java-dist/lib/<your system's architecture>/*_{jni,java}.*
         |__ libarrow_dataset_jni.dylib
         |__ libarrow_orc_jni.dylib
         |__ libgandiva_jni.dylib
@@ -145,7 +145,7 @@ Maven
 
         $ cd arrow/java
         $ mvn generate-resources -Pgenerate-libs-jni-windows -N
-        $ dir "../java-dist/bin"
+        $ dir "../java-dist/bin/x86_64"
         |__ arrow_dataset_jni.dll
 
 CMake
@@ -164,7 +164,7 @@ CMake
             -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF \
             -DBUILD_TESTING=OFF \
             -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_INSTALL_LIBDIR=lib \
+            -DCMAKE_INSTALL_LIBDIR=lib/<your system's architecture> \
             -DCMAKE_INSTALL_PREFIX=java-dist
         $ cmake --build java-cdata --target install --config Release
         $ ls -latr java-dist/lib
@@ -184,7 +184,7 @@ CMake
             -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF ^
             -DBUILD_TESTING=OFF ^
             -DCMAKE_BUILD_TYPE=Release ^
-            -DCMAKE_INSTALL_LIBDIR=lib ^
+            -DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^
             -DCMAKE_INSTALL_PREFIX=java-dist
         $ cmake --build java-cdata --target install --config Release
         $ dir "java-dist/bin"
@@ -219,7 +219,7 @@ CMake
             -DARROW_S3=ON \
             -DARROW_USE_CCACHE=ON \
             -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_INSTALL_LIBDIR=lib \
+            -DCMAKE_INSTALL_LIBDIR=lib/<your system's architecture> \
             -DCMAKE_INSTALL_PREFIX=java-dist \
             -DCMAKE_UNITY_BUILD=ON
         $ cmake --build cpp-jni --target install --config Release
@@ -230,11 +230,11 @@ CMake
             -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON \
             -DBUILD_TESTING=OFF \
             -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_INSTALL_LIBDIR=lib \
+            -DCMAKE_INSTALL_LIBDIR=lib/<your system's architecture> \
             -DCMAKE_INSTALL_PREFIX=java-dist \
             -DCMAKE_PREFIX_PATH=$PWD/java-dist
         $ cmake --build java-jni --target install --config Release
-        $ ls -latr java-dist/lib/*_{jni,java}.*
+        $ ls -latr java-dist/lib/<your system's architecture>/*_{jni,java}.*
         |__ libarrow_dataset_jni.dylib
         |__ libarrow_orc_jni.dylib
         |__ libgandiva_jni.dylib
@@ -265,7 +265,7 @@ CMake
             -DARROW_WITH_ZLIB=ON ^
             -DARROW_WITH_ZSTD=ON ^
             -DCMAKE_BUILD_TYPE=Release ^
-            -DCMAKE_INSTALL_LIBDIR=lib ^
+            -DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^
             -DCMAKE_INSTALL_PREFIX=java-dist ^
             -DCMAKE_UNITY_BUILD=ON ^
             -GNinja
@@ -282,7 +282,7 @@ CMake
             -DARROW_JAVA_JNI_ENABLE_PLASMA=OFF ^
             -DBUILD_TESTING=OFF ^
             -DCMAKE_BUILD_TYPE=Release ^
-            -DCMAKE_INSTALL_LIBDIR=lib ^
+            -DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^
             -DCMAKE_INSTALL_PREFIX=java-dist ^
             -DCMAKE_PREFIX_PATH=$PWD/java-dist
         $ cmake --build java-jni --target install --config Release
@@ -296,7 +296,7 @@ Archery
 
     $ cd arrow
     $ archery docker run java-jni-manylinux-2014
-    $ ls -latr java-dist/
+    $ ls -latr java-dist/<your system's architecture>/
     |__ libarrow_cdata_jni.so
     |__ libarrow_dataset_jni.so
     |__ libarrow_orc_jni.so
diff --git a/java/Brewfile b/java/Brewfile
new file mode 100644
index 00000000000..af6bd65615d
--- /dev/null
+++ b/java/Brewfile
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+brew "openjdk@11"
+brew "sccache"
diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java
index 600569be7c7..f7b839ece06 100644
--- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java
+++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java
@@ -23,6 +23,7 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
+import java.util.Locale;
 
 /**
  * Helper class for JNI related operations.
@@ -37,18 +38,34 @@ static void loadOrcAdapterLibraryFromJar()
           throws IOException, IllegalAccessException {
     synchronized (OrcJniUtils.class) {
       if (!isLoaded) {
-        final String libraryToLoad = System.mapLibraryName(LIBRARY_NAME);
-        final File libraryFile = moveFileFromJarToTemp(
-                System.getProperty("java.io.tmpdir"), libraryToLoad);
+        final String libraryToLoad =
+            getNormalizedArch() + File.separator + System.mapLibraryName(LIBRARY_NAME);
+        final File libraryFile =
+            moveFileFromJarToTemp(System.getProperty("java.io.tmpdir"), libraryToLoad, LIBRARY_NAME);
         System.load(libraryFile.getAbsolutePath());
         isLoaded = true;
       }
     }
   }
 
-  private static File moveFileFromJarToTemp(final String tmpDir, String libraryToLoad)
+  private static String getNormalizedArch() {
+    String arch = System.getProperty("os.arch").toLowerCase(Locale.US);
+    switch (arch) {
+      case "amd64":
+        arch = "x86_64";
+        break;
+      case "aarch64":
+        arch = "aarch_64";
+        break;
+      default:
+        break;
+    }
+    return arch;
+  }
+
+  private static File moveFileFromJarToTemp(final String tmpDir, String libraryToLoad, String libraryName)
           throws IOException {
-    final File temp = File.createTempFile(tmpDir, libraryToLoad);
+    final File temp = File.createTempFile(tmpDir, libraryName);
     try (final InputStream is = OrcReaderJniWrapper.class.getClassLoader()
             .getResourceAsStream(libraryToLoad)) {
       if (is == null) {
diff --git a/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java b/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java
index ca061937cde..ed71b6db284 100644
--- a/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java
+++ b/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java
@@ -27,6 +27,7 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 
 /**
@@ -78,7 +79,8 @@ private synchronized void loadRemaining() {
   }
 
   private void load(String name) {
-    final String libraryToLoad = System.mapLibraryName(name);
+    final String libraryToLoad =
+        getNormalizedArch() + File.separator + System.mapLibraryName(name);
     try {
       File temp = File.createTempFile("jnilib-", ".tmp", new File(System.getProperty("java.io.tmpdir")));
       temp.deleteOnExit();
@@ -93,4 +95,19 @@ private void load(String name) {
       throw new IllegalStateException("error loading native libraries: " + e);
     }
   }
+
+  private String getNormalizedArch() {
+    String arch = System.getProperty("os.arch").toLowerCase(Locale.US);
+    switch (arch) {
+      case "amd64":
+        arch = "x86_64";
+        break;
+      case "aarch64":
+        arch = "aarch_64";
+        break;
+      default:
+        break;
+    }
+    return arch;
+  }
 }
diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java
index dcbc4978f06..ffdd5e5273e 100644
--- a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java
+++ b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java
@@ -27,6 +27,7 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 
 /**
@@ -76,7 +77,8 @@ private synchronized void loadRemaining() {
   }
 
   private void load(String name) {
-    final String libraryToLoad = System.mapLibraryName(name);
+    final String libraryToLoad =
+        getNormalizedArch() + File.separator + System.mapLibraryName(name);
     try {
       File temp = File.createTempFile("jnilib-", ".tmp", new File(System.getProperty("java.io.tmpdir")));
       temp.deleteOnExit();
@@ -92,4 +94,19 @@ private void load(String name) {
       throw new IllegalStateException("error loading native libraries: " + e);
     }
   }
+
+  private String getNormalizedArch() {
+    String arch = System.getProperty("os.arch").toLowerCase(Locale.US);
+    switch (arch) {
+      case "amd64":
+        arch = "x86_64";
+        break;
+      case "aarch64":
+        arch = "aarch_64";
+        break;
+      default:
+        break;
+    }
+    return arch;
+  }
 }
diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java
index 676956a34ae..01c068f2926 100644
--- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java
+++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java
@@ -24,6 +24,7 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
+import java.util.Locale;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
@@ -69,15 +70,30 @@ private static JniLoader setupInstance() throws GandivaException {
 
   private static void loadGandivaLibraryFromJar(final String tmpDir)
           throws IOException, GandivaException {
-    final String libraryToLoad = System.mapLibraryName(LIBRARY_NAME);
-    final File libraryFile = moveFileFromJarToTemp(tmpDir, libraryToLoad);
+    final String libraryToLoad =
+        getNormalizedArch() + File.separator + System.mapLibraryName(LIBRARY_NAME);
+    final File libraryFile = moveFileFromJarToTemp(tmpDir, libraryToLoad, LIBRARY_NAME);
     System.load(libraryFile.getAbsolutePath());
   }
 
+  private static String getNormalizedArch() {
+    String arch = System.getProperty("os.arch").toLowerCase(Locale.US);
+    switch (arch) {
+      case "amd64":
+        arch = "x86_64";
+        break;
+      case "aarch64":
+        arch = "aarch_64";
+        break;
+      default:
+        break;
+    }
+    return arch;
+  }
 
-  private static File moveFileFromJarToTemp(final String tmpDir, String libraryToLoad)
+  private static File moveFileFromJarToTemp(final String tmpDir, String libraryToLoad, String libraryName)
           throws IOException, GandivaException {
-    final File temp = setupFile(tmpDir, libraryToLoad);
+    final File temp = setupFile(tmpDir, libraryName);
     try (final InputStream is = JniLoader.class.getClassLoader()
             .getResourceAsStream(libraryToLoad)) {
       if (is == null) {
diff --git a/java/plasma/pom.xml b/java/plasma/pom.xml
index f837d692f20..d6ae115dcd9 100644
--- a/java/plasma/pom.xml
+++ b/java/plasma/pom.xml
@@ -20,7 +20,19 @@
     <name>Arrow Plasma Client</name>
     <description>(DEPRECATED) Java client for the Plasma object store.
         This is deprecated since 10.0.0. This will be removed from 12.0.0 or so.</description>
+    <properties>
+        <arrow.cpp.build.dir>../../../cpp/release-build</arrow.cpp.build.dir>
+    </properties>
     <build>
+        <resources>
+            <resource>
+                <directory>${arrow.cpp.build.dir}</directory>
+                <includes>
+                    <include>**/*plasma_java.*</include>
+                </includes>
+            </resource>
+        </resources>
+
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
diff --git a/java/pom.xml b/java/pom.xml
index edc92422279..78182242adf 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -940,7 +940,7 @@
                     -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF
                     -DBUILD_TESTING=OFF
                     -DCMAKE_BUILD_TYPE=Release
-                    -DCMAKE_INSTALL_LIBDIR=lib
+                    -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch}
                     -DCMAKE_INSTALL_PREFIX=${arrow.c.jni.dist.dir}
                   </commandlineArgs>
                   <workingDirectory>../</workingDirectory>
@@ -1014,7 +1014,7 @@
                     -DARROW_S3=ON
                     -DARROW_USE_CCACHE=ON
                     -DCMAKE_BUILD_TYPE=Release
-                    -DCMAKE_INSTALL_LIBDIR=lib
+                    -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch}
                     -DCMAKE_INSTALL_PREFIX=java-dist
                     -DCMAKE_UNITY_BUILD=ON
                   </commandlineArgs>
@@ -1056,7 +1056,7 @@
                     -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON
                     -DBUILD_TESTING=OFF
                     -DCMAKE_BUILD_TYPE=Release
-                    -DCMAKE_INSTALL_LIBDIR=lib
+                    -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch}
                     -DCMAKE_INSTALL_PREFIX=${arrow.dataset.jni.dist.dir}
                     -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist
                   </commandlineArgs>
@@ -1130,7 +1130,7 @@
                     -DARROW_WITH_ZLIB=ON
                     -DARROW_WITH_ZSTD=ON
                     -DCMAKE_BUILD_TYPE=Release
-                    -DCMAKE_INSTALL_LIBDIR=lib
+                    -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch}
                     -DCMAKE_INSTALL_PREFIX=java-dist
                     -DCMAKE_UNITY_BUILD=ON
                     -GNinja
@@ -1173,7 +1173,7 @@
                     -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON
                     -DBUILD_TESTING=OFF
                     -DCMAKE_BUILD_TYPE=Release
-                    -DCMAKE_INSTALL_LIBDIR=lib
+                    -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch}
                     -DCMAKE_INSTALL_PREFIX=${arrow.dataset.jni.dist.dir}
                     -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist
                   </commandlineArgs>