Skip to content

Commit aa0c921

Browse files
koukou
committed
GH-500: [Release] Add support for releasing .jar including JNI binaries
Fixes GH-500. Merge `test_jni.yml` to `rc.yml` so that .jar including JNI binaries can be released. ccache support is also enabled for fast CI.
1 parent 3878927 commit aa0c921

File tree

10 files changed

+393
-343
lines changed

10 files changed

+393
-343
lines changed

‎.github/workflows/rc.yml‎

Lines changed: 279 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ concurrency:
3232
permissions:
3333
contents: read
3434
jobs:
35-
archive:
36-
name: Archive
35+
source:
36+
name: Source
3737
runs-on: ubuntu-latest
3838
timeout-minutes: 5
3939
steps:
@@ -70,15 +70,269 @@ jobs:
7070
- name: Audit
7171
run: |
7272
dev/release/run_rat.sh "${TAR_GZ}"
73-
- uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
73+
- name: Upload source archive
74+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
7475
with:
75-
name: archive
76+
name: release-source
7677
path: |
7778
apache-arrow-java-*
79+
jni-ubuntu:
80+
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
81+
runs-on: ${{ matrix.platform.runs_on }}
82+
needs:
83+
- source
84+
strategy:
85+
fail-fast: false
86+
matrix:
87+
platform:
88+
- runs_on: ubuntu-latest
89+
arch: "x86_64"
90+
archery_arch: "amd64"
91+
env:
92+
# architecture name used for archery build
93+
ARCH: ${{ matrix.platform.archery_arch }}
94+
DOCKER_VOLUME_PREFIX: .docker/
95+
permissions:
96+
contents: read
97+
packages: write
98+
steps:
99+
- name: Download source archive
100+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
101+
with:
102+
name: release-source
103+
- name: Extract Download the latest Apache Arrow C++
104+
run: |
105+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
106+
- name: Download the latest Apache Arrow C++
107+
run: |
108+
ci/scripts/download_cpp.sh
109+
- name: Checkout apache/arrow-testing
110+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
111+
with:
112+
repository: apache/arrow-testing
113+
path: arrow/testing
114+
- name: Checkout apache/parquet-testing
115+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
116+
with:
117+
repository: apache/parquet-testing
118+
path: arrow/cpp/submodules/parquet-testing
119+
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
120+
with:
121+
registry: ghcr.io
122+
username: ${{ github.actor }}
123+
password: ${{ secrets.GITHUB_TOKEN }}
124+
- name: Cache
125+
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
126+
with:
127+
path: .docker
128+
key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }}
129+
restore-keys: jni-linux-${{ matrix.platform.arch }}-
130+
- name: Build C++ libraries
131+
run: |
132+
docker compose run vcpkg-jni
133+
- name: Push Docker image
134+
if: success() && github.event_name == 'push' && github.repository == 'apache/arrow-java' && github.ref_name == 'main'
135+
run: |
136+
docker compose push vcpkg-jni
137+
- name: Compress into single artifact to keep directory structure
138+
run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/
139+
- name: Upload artifacts
140+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
141+
with:
142+
name: jni-linux-${{ matrix.platform.arch }}
143+
path: jni-linux-${{ matrix.platform.arch }}.tar.gz
144+
jni-macos:
145+
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
146+
runs-on: ${{ matrix.platform.runs_on }}
147+
needs:
148+
- source
149+
strategy:
150+
fail-fast: false
151+
matrix:
152+
platform:
153+
- { runs_on: macos-13, arch: "x86_64"}
154+
- { runs_on: macos-14, arch: "aarch_64" }
155+
env:
156+
MACOSX_DEPLOYMENT_TARGET: "14.0"
157+
steps:
158+
- name: Download source archive
159+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
160+
with:
161+
name: release-source
162+
- name: Extract Download the latest Apache Arrow C++
163+
run: |
164+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
165+
# We need 19.0.0 for latest Boost support
166+
- name: Download the latest RC Apache Arrow C++
167+
run: |
168+
ci/scripts/download_cpp.sh latest-rc
169+
- name: Checkout apache/arrow-testing
170+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
171+
with:
172+
repository: apache/arrow-testing
173+
path: arrow/testing
174+
- name: Checkout apache/parquet-testing
175+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
176+
with:
177+
repository: apache/parquet-testing
178+
path: arrow/cpp/submodules/parquet-testing
179+
- name: Set up Python
180+
uses: actions/setup-python@v5
181+
with:
182+
cache: 'pip'
183+
python-version: 3.12
184+
- name: Install Archery
185+
run: pip install -e arrow/dev/archery[all]
186+
- name: Install dependencies
187+
run: |
188+
# We want to use llvm@14 to avoid shared z3
189+
# dependency. llvm@14 doesn't depend on z3 and llvm depends
190+
# on z3. And Homebrew's z3 provides only shared library. It
191+
# doesn't provides static z3 because z3's CMake doesn't accept
192+
# building both shared and static libraries at once.
193+
# See also: Z3_BUILD_LIBZ3_SHARED in
194+
# https://github.com/Z3Prover/z3/blob/master/README-CMake.md
195+
#
196+
# If llvm is installed, Apache Arrow C++ uses llvm rather than
197+
# llvm@14 because llvm is newer than llvm@14.
198+
brew uninstall llvm || :
199+
200+
# Ensure updating python@XXX with the "--overwrite" option.
201+
# If python@XXX is updated without "--overwrite", it causes
202+
# a conflict error. Because Python 3 installed not by
203+
# Homebrew exists in /usr/local on GitHub Actions. If
204+
# Homebrew's python@XXX is updated without "--overwrite", it
205+
# tries to replace /usr/local/bin/2to3 and so on and causes
206+
# a conflict error.
207+
brew update
208+
for python_package in $(brew list | grep python@); do
209+
brew install --overwrite ${python_package}
210+
done
211+
brew install --overwrite python
212+
213+
if [ "$(uname -m)" = "arm64" ]; then
214+
# pkg-config formula is deprecated but it's still installed
215+
# in GitHub Actions runner now. We can remove this once
216+
# pkg-config formula is removed from GitHub Actions runner.
217+
brew uninstall pkg-config || :
218+
brew uninstall [email protected] || :
219+
fi
220+
221+
brew bundle --file=arrow/cpp/Brewfile
222+
# We want to link aws-sdk-cpp statically but Homebrew's
223+
# aws-sdk-cpp provides only shared library. If we have
224+
# Homebrew's aws-sdk-cpp, our build mix Homebrew's
225+
# aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
226+
# aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
227+
brew uninstall aws-sdk-cpp
228+
# We want to use bundled RE2 for static linking. If
229+
# Homebrew's RE2 is installed, its header file may be used.
230+
# We uninstall Homebrew's RE2 to ensure using bundled RE2.
231+
brew uninstall grpc || : # gRPC depends on RE2
232+
brew uninstall [email protected] || : # gRPC 1.54 may be installed too
233+
brew uninstall re2
234+
# We want to use bundled Protobuf for static linking. If
235+
# Homebrew's Protobuf is installed, its library file may be
236+
# used on test We uninstall Homebrew's Protobuf to ensure using
237+
# bundled Protobuf.
238+
brew uninstall protobuf
239+
240+
brew bundle --file=Brewfile
241+
- name: Prepare ccache
242+
run: |
243+
echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV}
244+
- name: Cache ccache
245+
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
246+
with:
247+
path: ccache
248+
key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }}
249+
restore-keys: jni-macos-${{ matrix.platform.arch }}-
250+
- name: Build C++ libraries
251+
run: |
252+
set -e
253+
# make brew Java available to CMake
254+
export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
255+
ci/scripts/jni_macos_build.sh . arrow build jni
256+
- name: Compress into single artifact to keep directory structure
257+
run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/
258+
- name: Upload artifacts
259+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
260+
with:
261+
name: jni-macos-${{ matrix.platform.arch }}
262+
path: jni-macos-${{ matrix.platform.arch }}.tar.gz
263+
binaries:
264+
name: Binaries
265+
runs-on: ubuntu-latest
266+
needs:
267+
- jni-ubuntu
268+
- jni-macos
269+
steps:
270+
- name: Download artifacts
271+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
272+
with:
273+
path: artifacts
274+
- name: Decompress artifacts
275+
run: |
276+
mv artifacts/*/*.tar.gz .
277+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
278+
tar -xvzf jni-linux-x86_64.tar.gz
279+
# tar -xvzf jni-linux-aarch_64.tar.gz
280+
tar -xvzf jni-macos-x86_64.tar.gz
281+
tar -xvzf jni-macos-aarch_64.tar.gz
282+
# tar -xvzf jni-windows.tar.gz
283+
- name: Test that shared libraries exist
284+
run: |
285+
set -x
286+
287+
test -f jni/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so
288+
test -f jni/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so
289+
test -f jni/arrow_orc_jni/x86_64/libarrow_orc_jni.so
290+
test -f jni/gandiva_jni/x86_64/libgandiva_jni.so
291+
292+
# test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so
293+
# test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so
294+
# test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.so
295+
# test -f jni/gandiva_jni/aarch_64/libgandiva_jni.so
296+
297+
test -f jni/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib
298+
test -f jni/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib
299+
test -f jni/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib
300+
test -f jni/gandiva_jni/x86_64/libgandiva_jni.dylib
301+
302+
test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib
303+
test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib
304+
test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
305+
test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib
306+
307+
# test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
308+
# test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
309+
# test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll
310+
- name: Checkout apache/arrow-testing
311+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
312+
with:
313+
repository: apache/arrow-testing
314+
path: testing
315+
- name: Cache ~/.m2
316+
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
317+
with:
318+
path: ~/.m2
319+
key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }}
320+
restore-keys: binaries-build-
321+
- name: Build bundled JAR
322+
env:
323+
MAVEN_ARGS: >-
324+
--no-transfer-progress
325+
run: |
326+
ci/scripts/jni_full_build.sh . jni binaries
327+
- name: Upload artifacts
328+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
329+
with:
330+
name: release-binaries
331+
path: binaries/*
78332
verify:
79333
name: Verify
80334
needs:
81-
- archive
335+
- binaries
82336
runs-on: ${{ matrix.os }}
83337
strategy:
84338
fail-fast: false
@@ -87,15 +341,13 @@ jobs:
87341
- macos-latest
88342
- ubuntu-latest
89343
steps:
90-
- name: Checkout
91-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
92-
with:
93-
submodules: recursive
94-
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
344+
- name: Download release artifacts
345+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
95346
with:
96-
name: archive
347+
pattern: release-*
97348
- name: Verify
98349
run: |
350+
mv release-source/* ./
99351
tar_gz=$(echo apache-arrow-java-*.tar.gz)
100352
version=${tar_gz#apache-arrow-java-}
101353
version=${version%.tar.gz}
@@ -105,37 +357,40 @@ jobs:
105357
else
106358
rc=$(date +%Y%m%d)
107359
fi
108-
VERIFY_DEFAULT=0 \
109-
VERIFY_SOURCE=1 \
110-
dev/release/verify_rc.sh "${version}" "${rc}"
360+
tar -xf ${tar_gz}
361+
export VERIFY_DEFAULT=0
362+
export VERIFY_BINARY=1
363+
export VERIFY_SOURCE=1
364+
cd apache-arrow-java-*
365+
mv ../${tar_gz}* ./
366+
mv ../release-binaries binaries
367+
dev/release/verify_rc.sh "${version}" "${rc}"
111368
upload:
112369
name: Upload
113-
if: github.ref_type == 'tag'
370+
# if: github.ref_type == 'tag'
114371
needs:
115372
- verify
116373
runs-on: ubuntu-latest
117374
permissions:
118375
contents: write
119376
steps:
120-
- name: Checkout
121-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
122-
with:
123-
submodules: recursive
124-
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
377+
- name: Download release artifacts
378+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
125379
with:
126-
name: archive
380+
pattern: release-*
381+
path: artifacts
127382
- name: Upload
128383
run: |
129384
# GH-499: How to create release notes?
130385
version=${GITHUB_REF_NAME%-rc*}
131386
version=${version#v}
132387
rc=${GITHUB_REF_NAME#*-rc}
133-
gh release create ${GITHUB_REF_NAME} \
388+
echo gh release create ${GITHUB_REF_NAME} \
134389
--generate-notes \
135390
--prerelease \
391+
--repo ${{ github.repository }} \
136392
--title "Apache Arrow Java ${version} RC${rc}" \
137393
--verify-tag \
138-
apache-arrow-java-*.tar.gz \
139-
apache-arrow-java-*.tar.gz.sha*
394+
artifacts/*/*
140395
env:
141396
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

0 commit comments

Comments
 (0)