diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml
index 71cb885946..3d6120dc24 100644
--- a/.github/workflows/rc.yml
+++ b/.github/workflows/rc.yml
@@ -394,17 +394,24 @@ jobs:
           path: ~/.m2
           key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }}
           restore-keys: binaries-build-
-      - name: Build bundled JAR
-        env:
-          MAVEN_ARGS: >-
-            --no-transfer-progress
+      - name: Build bundled JAR and docs
         run: |
           ci/scripts/jni_full_build.sh . jni binaries
-      - name: Upload artifacts
+      - name: Prepare docs
+        run: |
+          mkdir -p docs
+          cp -a target/site/apidocs docs/reference
+          tar -cvzf docs.tar.gz docs
+      - name: Upload binaries
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: release-binaries
           path: binaries/*
+      - name: Upload docs
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        with:
+          name: release-docs
+          path: docs.tar.gz
   verify:
     name: Verify
     needs:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a9ed7eb68f..3ea5d8a4b6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -66,7 +66,7 @@ jobs:
         uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
         with:
           path: .docker
-          key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('**/compose.yaml', '**/pom.xml') }}
+          key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }}
           restore-keys: maven-${{ matrix.jdk }}-${{ matrix.maven }}-
       - name: Execute Docker Build
         env:
@@ -108,12 +108,12 @@ jobs:
         shell: bash
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
-        run: ci/scripts/build.sh $(pwd) $(pwd)/build
+        run: ci/scripts/build.sh . build jni
       - name: Test
         shell: bash
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
-        run: ci/scripts/test.sh $(pwd) $(pwd)/build
+        run: ci/scripts/test.sh . build jni
 
   windows:
     name: AMD64 Windows Server 2022 Java JDK ${{ matrix.jdk }}
@@ -139,12 +139,12 @@ jobs:
         shell: bash
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
-        run: ci/scripts/build.sh $(pwd) $(pwd)/build
+        run: ci/scripts/build.sh . build jni
       - name: Test
         shell: bash
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
-        run: ci/scripts/test.sh $(pwd) $(pwd)/build
+        run: ci/scripts/test.sh . build jni
 
   integration:
     name: AMD64 integration
@@ -184,7 +184,7 @@ jobs:
         with:
           path: .docker
           key: integration-conda-${{ hashFiles('cpp/**') }}
-          restore-keys: conda-
+          restore-keys: integration-conda-
       - name: Setup Python
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh
index 309f28126a..146a40cf7a 100755
--- a/ci/scripts/build.sh
+++ b/ci/scripts/build.sh
@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -eo pipefail
+set -euo pipefail
 
 if [[ "${ARROW_JAVA_BUILD:-ON}" != "ON" ]]; then
   exit
@@ -27,11 +27,9 @@ source_dir=${1}
 build_dir=${2}
 java_jni_dist_dir=${3}
 
-: "${BUILD_DOCS_JAVA:=OFF}"
-
 mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
 
-if [ "$ARROW_JAVA_SKIP_GIT_PLUGIN" ]; then
+if [ "${ARROW_JAVA_SKIP_GIT_PLUGIN:-OFF}" = "ON" ]; then
   mvn="${mvn} -Dmaven.gitcommitid.skip=true"
 fi
 
@@ -61,27 +59,40 @@ done
 
 pushd "${build_dir}"
 
-if [ "${ARROW_JAVA_SHADE_FLATBUFFERS}" == "ON" ]; then
+# TODO: ARROW_JAVA_SHADE_FLATBUFFERS isn't used for our artifacts. Do
+# we need this?
+# See also:
+# * https://github.com/apache/arrow/issues/22021
+# * https://github.com/apache/arrow-java/issues/67
+if [ "${ARROW_JAVA_SHADE_FLATBUFFERS:-OFF}" == "ON" ]; then
   mvn="${mvn} -Pshade-flatbuffers"
 fi
 
-if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
+if [ "${ARROW_JAVA_CDATA:-OFF}" = "ON" ]; then
   mvn="${mvn} -Darrow.c.jni.dist.dir=${java_jni_dist_dir} -Parrow-c-data"
 fi
 
-if [ "${ARROW_JAVA_JNI}" = "ON" ]; then
+if [ "${ARROW_JAVA_JNI:-OFF}" = "ON" ]; then
   mvn="${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni"
 fi
 
 # Use `2 * ncores` threads
 ${mvn} -T 2C clean install
 
-if [ "${BUILD_DOCS_JAVA}" == "ON" ]; then
-  # HTTP pooling is turned of to avoid download issues https://issues.apache.org/jira/browse/ARROW-11633
-  # GH-43378: Maven site plugins not compatible with multithreading
-  mkdir -p "${build_dir}"/docs/java/reference
-  ${mvn} -Dcheckstyle.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false clean install site
-  rsync -a target/site/apidocs/ "${build_dir}"/docs/java/reference
+if [ "${ARROW_JAVA_BUILD_DOCS:-OFF}" == "ON" ]; then
+  # HTTP pooling is turned off to avoid download issues:
+  # https://github.com/apache/arrow/issues/27496
+  #
+  # Maven site plugins not compatible with multithreading:
+  # https://github.com/apache/arrow/issues/43378
+  ${mvn} \
+    -Dcheckstyle.skip=true \
+    -Dhttp.keepAlive=false \
+    -Dmaven.wagon.http.pool=false \
+    site
+  rm -rf docs/reference
+  mkdir -p docs
+  cp -a target/site/apidocs/ docs/reference
 fi
 
 popd
diff --git a/ci/scripts/jni_full_build.sh b/ci/scripts/jni_full_build.sh
index 49cbdc138f..e9ad0ddbda 100755
--- a/ci/scripts/jni_full_build.sh
+++ b/ci/scripts/jni_full_build.sh
@@ -17,15 +17,21 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -eu
+set -euo pipefail
 
+# shellcheck source=ci/scripts/util_log.sh
+. "$(dirname "${0}")/util_log.sh"
+
+github_actions_group_begin "Prepare arguments"
 source_dir="$(cd "${1}" && pwd)"
 jni_build_dir="$(cd "${2}" && pwd)"
 dist_dir="${3}"
 rm -rf "${dist_dir}"
 mkdir -p "${dist_dir}"
 dist_dir="$(cd "${dist_dir}" && pwd)"
+github_actions_group_end
 
+github_actions_group_begin "Clear old artifacts"
 # Ensure that there is no old artifacts inside the maven repository
 maven_repo=~/.m2/repository/org/apache/arrow
 if [ -d "$maven_repo" ]; then
@@ -34,7 +40,9 @@ if [ -d "$maven_repo" ]; then
     -exec echo {} ";" \
     -exec rm -rf {} ";"
 fi
+github_actions_group_end
 
+github_actions_group_begin "Generate dummy GPG key"
 # Generate dummy GPG key for -Papache-release.
 # -Papache-release generates signs (*.asc) of artifacts.
 # We don't use these signs in our release process.
@@ -46,19 +54,37 @@ fi
   echo "%no-protection"
 ) |
   gpg --full-generate-key --batch
+github_actions_group_end
 
 pushd "${source_dir}"
+github_actions_group_begin "Build .jar"
 # build the entire project
-mvn clean \
-  install \
+mvn \
+  --no-transfer-progress \
+  -Darrow.c.jni.dist.dir="${jni_build_dir}" \
+  -Darrow.cpp.build.dir="${jni_build_dir}" \
   -Papache-release \
   -Parrow-c-data \
   -Parrow-jni \
-  -Darrow.cpp.build.dir="${jni_build_dir}" \
+  clean \
+  install
+github_actions_group_end
+github_actions_group_begin "Build docs"
+# build docs
+mvn \
+  --no-transfer-progress \
   -Darrow.c.jni.dist.dir="${jni_build_dir}" \
-  --no-transfer-progress
+  -Darrow.cpp.build.dir="${jni_build_dir}" \
+  -Dcheckstyle.skip=true \
+  -Dhttp.keepAlive=false \
+  -Dmaven.wagon.http.pool=false \
+  -Parrow-c-data \
+  -Parrow-jni \
+  site
+github_actions_group_end
 popd
 
+github_actions_group_begin "Prepare artifacts"
 # copy all jar, zip and pom files to the distribution folder
 find ~/.m2/repository/org/apache/arrow \
   "(" \
@@ -75,3 +101,4 @@ for artifact in "${dist_dir}"/*; do
   sha256sum "${artifact}" >"${artifact}.sha256"
   sha512sum "${artifact}" >"${artifact}.sha512"
 done
+github_actions_group_end
diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh
index f65f630182..cacc20034e 100755
--- a/ci/scripts/test.sh
+++ b/ci/scripts/test.sh
@@ -23,10 +23,14 @@ if [[ "${ARROW_JAVA_TEST:-ON}" != "ON" ]]; then
   exit
 fi
 
-source_dir=${1}
+source_dir="$(cd "${1}" && pwd)"
 build_dir=${2}
 java_jni_dist_dir=${3}
 
+if [ -d "${java_jni_dist_dir}" ]; then
+  java_jni_dist_dir="$(cd "${java_jni_dist_dir}" && pwd)"
+fi
+
 mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
 # Use `2 * ncores` threads
 mvn="${mvn} -T 2C"
diff --git a/compose.yaml b/compose.yaml
index 5decbfc86c..58a19676a1 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -47,8 +47,8 @@ services:
       - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
     command:
       /bin/bash -c "
-        /arrow-java/ci/scripts/build.sh /arrow-java /build &&
-        /arrow-java/ci/scripts/test.sh /arrow-java /build"
+        /arrow-java/ci/scripts/build.sh /arrow-java /build /jni &&
+        /arrow-java/ci/scripts/test.sh /arrow-java /build /jni"
 
   conda-jni-cdata:
     # Builds and tests just the C Data Interface JNI library and JARs.
@@ -103,11 +103,11 @@ services:
     volumes:
       - .:/arrow-java:delegated
       - ${ARROW_REPO_ROOT}:/arrow:delegated
-      - ${DOCKER_VOLUME_PREFIX}ccache:/ccache:delegated
+      - ${DOCKER_VOLUME_PREFIX}ccache-cache:/ccache:delegated
       - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
     environment:
       ARROW_JAVA_CDATA: "ON"
       CCACHE_DIR: "/ccache"
     command:
       ["git config --global --add safe.directory /arrow-java && \
-        /arrow-java/ci/scripts/jni_manylinux_build.sh /arrow-java /arrow /build /arrow-java/jni"]
+        /arrow-java/ci/scripts/jni_manylinux_build.sh /arrow-java /arrow /build/java /arrow-java/jni"]