From cafcf2cb5c9938efbbab9264f5f78de320d93d70 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 26 Aug 2023 08:43:03 +0800 Subject: [PATCH 1/8] ci(bindings/java): enable auto staging JARs on Apache Nexus repository Signed-off-by: tison --- .editorconfig | 7 +- .github/workflows/release_java.yml | 131 +++++++++++++++++++++++++++++ licenserc.toml | 3 + scripts/merge_local_staging.sh | 40 +++++++++ 4 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/release_java.yml create mode 100644 scripts/merge_local_staging.sh diff --git a/.editorconfig b/.editorconfig index 77b65fb57d30..c14889663d5e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -38,13 +38,16 @@ indent_size = 2 [*.yml] indent_size = 2 +[*.toml] +indent_size = 2 + [*.json] indent_size = 2 -[*.{ts, tsx}] +[*.{ts,tsx}] indent_size = 2 -[*.{js, jsx}] +[*.{js,jsx}] indent_size = 2 [*.css] diff --git a/.github/workflows/release_java.yml b/.github/workflows/release_java.yml new file mode 100644 index 000000000000..ce038178a6fa --- /dev/null +++ b/.github/workflows/release_java.yml @@ -0,0 +1,131 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Bindings Java Release + +on: + push: + branches: + # FIXME test-only; remove before merge + - auto-release-java + tags: + # Staging JARs on Apache Nexus repository for RCs. Read more on + # https://opendal.apache.org/docs/contributing/release#release-maven-artifacts + - 'v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+' + workflow_dispatch: + +jobs: + stage-snapshot: + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + classifier: linux-x86_64 + - os: windows-latest + classifier: windows-x86_64 + - os: macos-latest + classifier: osx-x86_64 + - os: macos-latest + classifier: osx-aarch_64 + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 8 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '8' + cache: 'maven' + server-id: apache.releases.https + server-username: MAVEN_USERNAME + server-password: MAVEN_CENTRAL_TOKEN + gpg-private-key: ${{ secrets.GPG_SECRET_KEY }} + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Local staging + working-directory: bindings/java + run: | + mvn -Papache-release package verify org.sonatype.plugins:nexus-staging-maven-plugin:deploy "-Djni.classifier=${{ matrix.classifier }}" -DskipTests=true -DaltStagingDirectory=local-staging -DskipRemoteStaging=true "-DserverId=apache.releases.https" "-DnexusUrl=https://repository.apache.org" + env: + MAVEN_USERNAME: ${{ secrets.NEXUS_STAGE_DEPLOYER_USER }} + MAVEN_CENTRAL_TOKEN: ${{ secrets.NEXUS_STAGE_DEPLOYER_PW }} + - name: Upload local staging directory + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.classifier }}-local-staging + path: bindings/java/local-staging + if-no-files-found: error + + deploy-staged-snapshots: + runs-on: ubuntu-latest + needs: [stage-snapshot] + steps: + - uses: actions/checkout@v3 + with: + repository: apache/incubator-opendal + ref: ${{ inputs.ref }} + - name: Set up JDK 8 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '8' + cache: 'maven' + server-id: apache.releases.https + server-username: MAVEN_USERNAME + server-password: MAVEN_CENTRAL_TOKEN + gpg-private-key: ${{ secrets.GPG_SECRET_KEY }} + + - name: Prepare environment variables + run: echo "LOCAL_STAGING_DIR=$HOME/local-staging" >> $GITHUB_ENV + + - name: Download windows staging directory + uses: actions/download-artifact@v3 + with: + name: windows-x86_64-local-staging + path: ~/windows-x86_64-local-staging + - name: Download linux staging directory + uses: actions/download-artifact@v3 + with: + name: linux-x86_64-local-staging + path: ~/linux-x86_64-local-staging + - name: Download darwin staging directory + uses: actions/download-artifact@v3 + with: + name: osx-x86_64-local-staging + path: ~/osx-x86_64-local-staging + - name: Download darwin (aarch64) staging directory + uses: actions/download-artifact@v3 + with: + name: osx-aarch_64-local-staging + path: ~/osx-aarch_64-local-staging + + - uses: actions/checkout@v3 + with: + path: ci-opendal + - name: Merge staging repositories + working-directory: ci-opendal + run: bash ./scripts/merge_local_staging.sh $LOCAL_STAGING_DIR/staging ~/windows-x86_64-local-staging/staging ~/linux-x86_64-local-staging/staging ~/osx-x86_64-local-staging/staging ~/osx-aarch_64-local-staging/staging + + - name: Deploy local staged artifacts + if: ${{ github.event_name != 'pull_request' }} + working-directory: bindings/java + run: | + mvn org.sonatype.plugins:nexus-staging-maven-plugin:deploy-staged -DaltStagingDirectory=$LOCAL_STAGING_DIR -DskipStagingRepositoryClose=true -DserverId=apache.releases.https -DnexusUrl=https://repository.apache.org + env: + MAVEN_USERNAME: ${{ secrets.NEXUS_STAGE_DEPLOYER_USER }} + MAVEN_CENTRAL_TOKEN: ${{ secrets.NEXUS_STAGE_DEPLOYER_PW }} diff --git a/licenserc.toml b/licenserc.toml index f4ea72855069..7d83bcc2db32 100644 --- a/licenserc.toml +++ b/licenserc.toml @@ -35,4 +35,7 @@ excludes = [ # Documents "**/*.md", "**/*.mdx", + + # Third-party source files + "scripts/merge_local_staging.sh", ] diff --git a/scripts/merge_local_staging.sh b/scripts/merge_local_staging.sh new file mode 100644 index 000000000000..d28445df307c --- /dev/null +++ b/scripts/merge_local_staging.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# ---------------------------------------------------------------------------- +# Copyright 2021 The Netty Project +# +# The Netty Project licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# This file is copied from +# https://github.com/netty/netty-tcnative/blob/67a73e0c/.github/scripts/merge_local_staging.sh + +set -e +if [ "$#" -lt 2 ]; then + echo "Expected target directory and at least one local staging directory" + exit 1 +fi +TARGET=$1 + +for ((i=2; i<=$#; i++)) +do + DIR="${!i}" + SUB_DIR=$(ls -d "${DIR}"/* | awk -F / '{print $NF}') + + if [ ! -d "${TARGET}/${SUB_DIR}" ] + then + mkdir -p "${TARGET}/${SUB_DIR}" + fi + cat "${DIR}"/"${SUB_DIR}"/.index >> "${TARGET}/${SUB_DIR}"/.index + cp -r "${DIR}"/"${SUB_DIR}"/* "${TARGET}/${SUB_DIR}"/ +done From 799cb6b6169a577e0c519af10a5cff050148c3c3 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 26 Aug 2023 08:54:51 +0800 Subject: [PATCH 2/8] fix install protoc Signed-off-by: tison --- .github/workflows/bindings_java.yml | 6 ++++-- .github/workflows/release_java.yml | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bindings_java.yml b/.github/workflows/bindings_java.yml index f2584c837034..367042cd5a1f 100644 --- a/.github/workflows/bindings_java.yml +++ b/.github/workflows/bindings_java.yml @@ -66,8 +66,10 @@ jobs: distribution: 'temurin' java-version: '8' cache: 'maven' - - name: Set up protoc - run: sudo apt install -y protobuf-compiler + - name: Install Protoc + uses: arduino/setup-protoc@v2 + with: + version: "23.4" - name: Build and test working-directory: bindings/java # `install` is required to verify reproducible builds: diff --git a/.github/workflows/release_java.yml b/.github/workflows/release_java.yml index ce038178a6fa..49b75d12ddae 100644 --- a/.github/workflows/release_java.yml +++ b/.github/workflows/release_java.yml @@ -57,6 +57,15 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.10' + - name: Install Protoc + uses: arduino/setup-protoc@v2 + with: + version: "23.4" + - name: Verify reproducibility before staging + working-directory: bindings/java + run: | + ./mvnw clean install + ./mvnw verify artifact:compare - name: Local staging working-directory: bindings/java run: | From a22d4a58a22988d0cca3992525014b7431806cfc Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 26 Aug 2023 16:54:09 +0800 Subject: [PATCH 3/8] sftp is not available on Windows Signed-off-by: tison --- .github/workflows/bindings_java.yml | 8 ++- bindings/java/Cargo.toml | 88 ++++++++++++++++------------- 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/.github/workflows/bindings_java.yml b/.github/workflows/bindings_java.yml index 367042cd5a1f..47c03a0b4590 100644 --- a/.github/workflows/bindings_java.yml +++ b/.github/workflows/bindings_java.yml @@ -57,7 +57,13 @@ jobs: run: ./mvnw spotless:check test: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + - os: windows-latest + - os: macos-latest steps: - uses: actions/checkout@v3 - name: Set up JDK 8 diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index 94a495675ee4..7d94cd71bdff 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -41,44 +41,52 @@ tokio = { version = "1.28.1", features = ["full"] } [dependencies.opendal] workspace = true features = [ - "services-azblob", - "services-azdfs", - "services-cacache", - "services-cos", - "services-dashmap", - "services-dropbox", - "services-etcd", - # FIXME this requires a preinstalled fdb library - # "services-foundationdb", - "services-fs", - "services-ftp", - "services-gcs", - "services-gdrive", - "services-ghac", - # FIXME how to support HDFS services in bindings? - # "services-hdfs", - "services-http", - "services-ipfs", - "services-memcached", - "services-memory", - "services-mini-moka", - "services-moka", - "services-obs", - "services-onedrive", - "services-oss", - "services-persy", - "services-postgresql", - "services-redb", - "services-redis", - "services-redis-rustls", - "services-rocksdb", - "services-s3", - "services-sftp", - "services-sled", - "services-supabase", - "services-tikv", - "services-vercel-artifacts", - "services-wasabi", - "services-webdav", - "services-webhdfs", + "services-azblob", + "services-azdfs", + "services-cacache", + "services-cos", + "services-dashmap", + "services-dropbox", + "services-etcd", + # FIXME this requires a preinstalled fdb library + # "services-foundationdb", + "services-fs", + "services-ftp", + "services-gcs", + "services-gdrive", + "services-ghac", + # FIXME how to support HDFS services in bindings? + # "services-hdfs", + "services-http", + "services-ipfs", + "services-memcached", + "services-memory", + "services-mini-moka", + "services-moka", + "services-obs", + "services-onedrive", + "services-oss", + "services-persy", + "services-postgresql", + "services-redb", + "services-redis", + "services-redis-rustls", + "services-rocksdb", + "services-s3", + "services-sled", + "services-supabase", + "services-tikv", + "services-vercel-artifacts", + "services-wasabi", + "services-webdav", + "services-webhdfs", +] + +# This is not optimal. See also the Cargo issue: +# https://github.com/rust-lang/cargo/issues/1197#issuecomment-1641086954 +[target.'cfg(unix)'.dependencies.opendal] +workspace = true +features = [ + # Depend on "openssh" which depends on "tokio-pipe" that is unavailable on Windows. + "services-sftp", ] From ece1f40db478cac1b937f028735675121da4af58 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 26 Aug 2023 23:12:47 +0800 Subject: [PATCH 4/8] try empty MAVEN_GPG_PASSPHRASE Signed-off-by: tison --- .github/workflows/release_java.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release_java.yml b/.github/workflows/release_java.yml index 49b75d12ddae..514161254b83 100644 --- a/.github/workflows/release_java.yml +++ b/.github/workflows/release_java.yml @@ -54,6 +54,7 @@ jobs: server-username: MAVEN_USERNAME server-password: MAVEN_CENTRAL_TOKEN gpg-private-key: ${{ secrets.GPG_SECRET_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE - uses: actions/setup-python@v4 with: python-version: '3.10' @@ -73,6 +74,7 @@ jobs: env: MAVEN_USERNAME: ${{ secrets.NEXUS_STAGE_DEPLOYER_USER }} MAVEN_CENTRAL_TOKEN: ${{ secrets.NEXUS_STAGE_DEPLOYER_PW }} + MAVEN_GPG_PASSPHRASE: "" - name: Upload local staging directory uses: actions/upload-artifact@v3 with: @@ -98,6 +100,7 @@ jobs: server-username: MAVEN_USERNAME server-password: MAVEN_CENTRAL_TOKEN gpg-private-key: ${{ secrets.GPG_SECRET_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE - name: Prepare environment variables run: echo "LOCAL_STAGING_DIR=$HOME/local-staging" >> $GITHUB_ENV @@ -138,3 +141,4 @@ jobs: env: MAVEN_USERNAME: ${{ secrets.NEXUS_STAGE_DEPLOYER_USER }} MAVEN_CENTRAL_TOKEN: ${{ secrets.NEXUS_STAGE_DEPLOYER_PW }} + MAVEN_GPG_PASSPHRASE: "" From 8816d40b764958d9a3050d3b18d624d0d9f15371 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 26 Aug 2023 23:15:10 +0800 Subject: [PATCH 5/8] workaround rate limit Signed-off-by: tison --- .github/workflows/bindings_java.yml | 1 + .github/workflows/release_java.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/bindings_java.yml b/.github/workflows/bindings_java.yml index 47c03a0b4590..169b2935b2fe 100644 --- a/.github/workflows/bindings_java.yml +++ b/.github/workflows/bindings_java.yml @@ -76,6 +76,7 @@ jobs: uses: arduino/setup-protoc@v2 with: version: "23.4" + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build and test working-directory: bindings/java # `install` is required to verify reproducible builds: diff --git a/.github/workflows/release_java.yml b/.github/workflows/release_java.yml index 514161254b83..00d472b96db6 100644 --- a/.github/workflows/release_java.yml +++ b/.github/workflows/release_java.yml @@ -62,6 +62,7 @@ jobs: uses: arduino/setup-protoc@v2 with: version: "23.4" + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Verify reproducibility before staging working-directory: bindings/java run: | From c2e55e6722f2e0e54038fabd7809d2afc9eb0fdc Mon Sep 17 00:00:00 2001 From: tison Date: Sun, 27 Aug 2023 00:52:44 +0800 Subject: [PATCH 6/8] keep use mvnw and the same classifier Signed-off-by: tison --- .github/workflows/release_java.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_java.yml b/.github/workflows/release_java.yml index 00d472b96db6..4976a29a7b4b 100644 --- a/.github/workflows/release_java.yml +++ b/.github/workflows/release_java.yml @@ -66,12 +66,12 @@ jobs: - name: Verify reproducibility before staging working-directory: bindings/java run: | - ./mvnw clean install - ./mvnw verify artifact:compare + ./mvnw clean install "-Djni.classifier=${{ matrix.classifier }}" + ./mvnw verify artifact:compare "-Djni.classifier=${{ matrix.classifier }}" - name: Local staging working-directory: bindings/java run: | - mvn -Papache-release package verify org.sonatype.plugins:nexus-staging-maven-plugin:deploy "-Djni.classifier=${{ matrix.classifier }}" -DskipTests=true -DaltStagingDirectory=local-staging -DskipRemoteStaging=true "-DserverId=apache.releases.https" "-DnexusUrl=https://repository.apache.org" + ./mvnw -Papache-release package verify org.sonatype.plugins:nexus-staging-maven-plugin:deploy "-Djni.classifier=${{ matrix.classifier }}" -DskipTests=true -DaltStagingDirectory=local-staging -DskipRemoteStaging=true "-DserverId=apache.releases.https" "-DnexusUrl=https://repository.apache.org" env: MAVEN_USERNAME: ${{ secrets.NEXUS_STAGE_DEPLOYER_USER }} MAVEN_CENTRAL_TOKEN: ${{ secrets.NEXUS_STAGE_DEPLOYER_PW }} @@ -138,7 +138,7 @@ jobs: if: ${{ github.event_name != 'pull_request' }} working-directory: bindings/java run: | - mvn org.sonatype.plugins:nexus-staging-maven-plugin:deploy-staged -DaltStagingDirectory=$LOCAL_STAGING_DIR -DskipStagingRepositoryClose=true -DserverId=apache.releases.https -DnexusUrl=https://repository.apache.org + ./mvnw org.sonatype.plugins:nexus-staging-maven-plugin:deploy-staged -DaltStagingDirectory=$LOCAL_STAGING_DIR -DskipStagingRepositoryClose=true -DserverId=apache.releases.https -DnexusUrl=https://repository.apache.org env: MAVEN_USERNAME: ${{ secrets.NEXUS_STAGE_DEPLOYER_USER }} MAVEN_CENTRAL_TOKEN: ${{ secrets.NEXUS_STAGE_DEPLOYER_PW }} From fb6c6d26048396c2a84ecd077ab9be2779b2ec4e Mon Sep 17 00:00:00 2001 From: tison Date: Sun, 27 Aug 2023 10:28:58 +0800 Subject: [PATCH 7/8] reproducibility is already tested in per PR CI Signed-off-by: tison --- .github/workflows/release_java.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/release_java.yml b/.github/workflows/release_java.yml index 4976a29a7b4b..d8260926dedb 100644 --- a/.github/workflows/release_java.yml +++ b/.github/workflows/release_java.yml @@ -63,11 +63,6 @@ jobs: with: version: "23.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Verify reproducibility before staging - working-directory: bindings/java - run: | - ./mvnw clean install "-Djni.classifier=${{ matrix.classifier }}" - ./mvnw verify artifact:compare "-Djni.classifier=${{ matrix.classifier }}" - name: Local staging working-directory: bindings/java run: | From b4841e432099735e666762c453286b0f5d05863f Mon Sep 17 00:00:00 2001 From: tison Date: Sun, 27 Aug 2023 11:22:04 +0800 Subject: [PATCH 8/8] drop scaffolding Signed-off-by: tison --- .github/workflows/release_java.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/release_java.yml b/.github/workflows/release_java.yml index d8260926dedb..4b9223f6ca94 100644 --- a/.github/workflows/release_java.yml +++ b/.github/workflows/release_java.yml @@ -19,9 +19,6 @@ name: Bindings Java Release on: push: - branches: - # FIXME test-only; remove before merge - - auto-release-java tags: # Staging JARs on Apache Nexus repository for RCs. Read more on # https://opendal.apache.org/docs/contributing/release#release-maven-artifacts