chore(ci): adjust build & test workflows (#17213)

databendlabs · Jan 8, 2025 · 1bf05c3 · 1bf05c3
1 parent 976858c
commit 1bf05c3
Show file tree

Hide file tree

Showing 40 changed files with 113 additions and 427 deletions.
diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml
@@ -69,11 +69,12 @@ runs:
       shell: bash
       run: |
         artifacts="meta,metactl,metabench,query,sqllogictests"
+        binaries=""
         for artifact in ${artifacts//,/ }; do
-          echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact
+          binaries="${binaries} --bin databend-$artifact"
         done
-        ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
+        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} ${binaries}
+        ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-*
 
     - name: Build Release for specific artifacts
       if: inputs.artifacts != 'all'
@@ -83,8 +84,8 @@ runs:
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
           cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact
+          ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
         done
-        ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
 
     - name: Check Binary Info
       shell: bash

diff --git a/.github/actions/setup_build_tool/action.yml b/.github/actions/setup_build_tool/action.yml
@@ -38,12 +38,12 @@ runs:
         EOF
 
         RUNNER_PROVIDER="${RUNNER_PROVIDER:-github}"
-        export SCCACHE_IDLE_TIMEOUT=0
         case ${RUNNER_PROVIDER} in
           aws)
             echo "setting up sccache for AWS S3..."
             cat <<EOF >>$BIN_LOCAL/build-tool
         export ENABLE_SCCACHE=true
+        export SCCACHE_IDLE_TIMEOUT=0
         export SCCACHE_BUCKET=databend-ci
         export SCCACHE_REGION=us-east-2
         export SCCACHE_S3_KEY_PREFIX="cache/sccache/"
@@ -57,6 +57,7 @@ runs:
             echo "setting up sccache for Google Cloud Storage..."
             cat <<EOF >>$BIN_LOCAL/build-tool
         export ENABLE_SCCACHE=true
+        export SCCACHE_IDLE_TIMEOUT=0
         export SCCACHE_GCS_RW_MODE=READ_WRITE
         export SCCACHE_GCS_BUCKET=databend-ci
         export SCCACHE_GCS_KEY_PREFIX="cache/sccache/"

diff --git a/.github/actions/setup_test/action.yml b/.github/actions/setup_test/action.yml
@@ -0,0 +1,44 @@
+name: "Setup For GNU Test"
+description: "Setup For GNU Test"
+inputs:
+  path:
+    description: "default to ./target/${BUILD_PROFILE}/"
+    required: false
+    default: ""
+  artifacts:
+    description: "Artifacts to download, only works with s3/gcs"
+    required: false
+    default: "meta,query"
+runs:
+  using: "composite"
+  steps:
+    - id: target
+      shell: bash
+      run: |
+        arch=$(uname -m)
+        target="${arch}-unknown-linux-gnu"
+        echo "target=${target}" >> $GITHUB_OUTPUT
+
+    - uses: ./.github/actions/setup_bendsql
+
+    - uses: ./.github/actions/artifact_download
+      with:
+        sha: ${{ github.sha }}
+        target: ${{ steps.target.outputs.target }}
+        category: full
+        artifacts: ${{ inputs.artifacts }}
+        path: ${{ inputs.path }}
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: "11"
+
+    - name: setup java env
+      shell: bash
+      run: |
+        echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${{ env.LD_LIBRARY_PATH }}" >> $GITHUB_ENV
diff --git a/.github/actions/test_cloud_sqllogic_standalone_linux/action.yml b/.github/actions/test_cloud_sqllogic_standalone_linux/action.yml
@@ -1,10 +1,6 @@
 name: "Test cloud control sqllogic Standalone"
 description: "Running cloud control sqllogic tests in standalone mode"
 inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
   dirs:
     description: "logic test suites dirs"
     required: true
@@ -21,11 +17,8 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
+    - uses: ./.github/actions/setup_test
       with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
         artifacts: sqllogictests,meta,query
 
     - name: Run sqllogic Tests with Standalone mode

diff --git a/.github/actions/test_compat_fuse/action.yml b/.github/actions/test_compat_fuse/action.yml
@@ -1,19 +1,10 @@
 name: "Test fuse-table format in an old query is compatible with new query"
 description: "Download old binaries and current binaries, write data with old query, read data with new query"
-inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
 runs:
   using: "composite"
   steps:
-    - uses: ./.github/actions/setup_bendsql
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
+    - uses: ./.github/actions/setup_test
       with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
         path: ./bins/current
         artifacts: sqllogictests,meta,query
 

diff --git a/.github/actions/test_compat_meta_meta/action.yml b/.github/actions/test_compat_meta_meta/action.yml
@@ -1,18 +1,10 @@
 name: "Test protocol compatibility between databend-meta servers"
 description: "Download old binaries and current binaries, write some data and then read data"
-inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
 runs:
   using: "composite"
   steps:
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
+    - uses: ./.github/actions/setup_test
       with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
         path: ./bins/current/bin
         artifacts: meta,metactl,metabench
 

diff --git a/.github/actions/test_compat_meta_query/action.yml b/.github/actions/test_compat_meta_query/action.yml
@@ -1,18 +1,10 @@
 name: "Test old metasrv is compatible with new query; new meta is compatible with old query"
 description: "Download old binaries and current binaries, run stateless test with old `meta` and latest `query`"
-inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
 runs:
   using: "composite"
   steps:
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
+    - uses: ./.github/actions/setup_test
       with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
         path: ./bins/current
         artifacts: sqllogictests,meta,query
 

diff --git a/.github/actions/test_ee_management_mode_linux/action.yml b/.github/actions/test_ee_management_mode_linux/action.yml
@@ -1,19 +1,9 @@
 name: "Test ee management mode"
 description: "Running stateless tests in management mode"
-inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
 runs:
   using: "composite"
   steps:
-    - uses: ./.github/actions/setup_bendsql
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
-      with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
+    - uses: ./.github/actions/setup_test
 
     - name: Run Stateful Tests with Standalone mode
       shell: bash

diff --git a/.github/actions/test_ee_sqllogic_standalone_linux/action.yml b/.github/actions/test_ee_sqllogic_standalone_linux/action.yml
@@ -1,10 +1,6 @@
 name: "Test ee sqllogic Standalone"
 description: "Running ee sqllogic tests in standalone mode"
 inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
   dirs:
     description: "logic test suites dirs"
     required: true
@@ -21,12 +17,8 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: ./.github/actions/setup_bendsql
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
+    - uses: ./.github/actions/setup_test
       with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
         artifacts: sqllogictests,meta,query
 
     - name: Run sqllogic Tests with Standalone mode

diff --git a/.github/actions/test_ee_standalone_background_linux/action.yml b/.github/actions/test_ee_standalone_background_linux/action.yml
@@ -1,19 +1,9 @@
 name: "Test Enterprise Background Service features Standalone"
 description: "Running stateless tests in standalone mode"
-inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
 runs:
   using: "composite"
   steps:
-    - uses: ./.github/actions/setup_bendsql
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
-      with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
+    - uses: ./.github/actions/setup_test
 
     - name: Minio Setup for (ubuntu-latest only)
       shell: bash

diff --git a/.github/actions/test_ee_standalone_fake_time_linux/action.yml b/.github/actions/test_ee_standalone_fake_time_linux/action.yml
@@ -1,19 +1,9 @@
 name: "Test Enterprise features Standalone"
 description: "Running stateless tests in standalone mode"
-inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
 runs:
   using: "composite"
   steps:
-    - uses: ./.github/actions/setup_bendsql
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
-      with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
+    - uses: ./.github/actions/setup_test
 
     - name: Minio Setup for (ubuntu-latest only)
       shell: bash

diff --git a/.github/actions/test_ee_standalone_linux/action.yml b/.github/actions/test_ee_standalone_linux/action.yml
@@ -1,19 +1,9 @@
 name: "Test Enterprise features Standalone"
 description: "Running stateless tests in standalone mode"
-inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
 runs:
   using: "composite"
   steps:
-    - uses: ./.github/actions/setup_bendsql
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
-      with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
+    - uses: ./.github/actions/setup_test
 
     - name: Minio Setup for (ubuntu-latest only)
       shell: bash

diff --git a/.github/actions/test_fuzz_standalone_linux/action.yml b/.github/actions/test_fuzz_standalone_linux/action.yml
@@ -1,20 +1,9 @@
 name: "Test Fuzz Standalone"
 description: "Running fuzz tests in standalone mode"
-inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
 runs:
   using: "composite"
   steps:
-    - uses: ./.github/actions/setup_bendsql
-
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
-      with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
+    - uses: ./.github/actions/setup_test
 
     - name: Test setup
       shell: bash

diff --git a/.github/actions/test_logs/action.yml b/.github/actions/test_logs/action.yml
@@ -1,20 +1,9 @@
 name: "Test collection query & profile logs for databend query"
 description: "Test collection query & profile logs for databend query"
-inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
 runs:
   using: "composite"
   steps:
-    - uses: ./.github/actions/setup_bendsql
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
-      with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
-        artifacts: meta,query
+    - uses: ./.github/actions/setup_test
 
     - name: Run logging Tests
       shell: bash

diff --git a/.github/actions/test_meta_cluster/action.yml b/.github/actions/test_meta_cluster/action.yml
@@ -1,18 +1,9 @@
 name: "Test databend-meta cluster"
 description: "Download binaries, setup cluster by joining and leaving"
-inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
 runs:
   using: "composite"
   steps:
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
-      with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
+    - uses: ./.github/actions/setup_test
 
     - name: Test databend-meta cluster
       shell: bash

diff --git a/.github/actions/test_metactl/action.yml b/.github/actions/test_metactl/action.yml
@@ -1,18 +1,10 @@
 name: "Test metactl"
 description: ""
-inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
 runs:
   using: "composite"
   steps:
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
+    - uses: ./.github/actions/setup_test
       with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
         artifacts: metactl,meta
 
     - name: Test metactl

diff --git a/.github/actions/test_sqllogic_cluster_linux/action.yml b/.github/actions/test_sqllogic_cluster_linux/action.yml
@@ -1,10 +1,6 @@
 name: "Test sqllogic Cluster Linux"
 description: "Running sqllogic tests in cluster mode"
 inputs:
-  target:
-    description: ""
-    required: true
-    default: "x86_64-unknown-linux-gnu"
   dirs:
     description: "logic test suites dirs"
     required: true
@@ -16,12 +12,8 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: ./.github/actions/setup_bendsql
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
+    - uses: ./.github/actions/setup_test
       with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
         artifacts: sqllogictests,meta,query
 
     - name: Run sqllogic Tests with Cluster mode