From 1bf05c384af79122f4f74abba2c4d484c6957f5b Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 8 Jan 2025 19:50:45 +0800 Subject: [PATCH] chore(ci): adjust build & test workflows (#17213) --- .github/actions/build_linux/action.yml | 9 +-- .github/actions/setup_build_tool/action.yml | 3 +- .github/actions/setup_test/action.yml | 44 ++++++++++++++ .../action.yml | 9 +-- .github/actions/test_compat_fuse/action.yml | 11 +--- .../actions/test_compat_meta_meta/action.yml | 10 +--- .../actions/test_compat_meta_query/action.yml | 10 +--- .../test_ee_management_mode_linux/action.yml | 12 +--- .../action.yml | 10 +--- .../action.yml | 12 +--- .../action.yml | 12 +--- .../test_ee_standalone_linux/action.yml | 12 +--- .../test_fuzz_standalone_linux/action.yml | 13 +---- .github/actions/test_logs/action.yml | 13 +---- .github/actions/test_meta_cluster/action.yml | 11 +--- .github/actions/test_metactl/action.yml | 10 +--- .../test_sqllogic_cluster_linux/action.yml | 10 +--- .../test_sqllogic_iceberg_tpch/action.yml | 11 +--- .../action.yml | 10 +--- .../actions/test_sqllogic_stage/action.yml | 11 +--- .../test_sqllogic_standalone_linux/action.yml | 12 +--- .../action.yml | 10 +--- .../test_sqllogic_standalone_macos/action.yml | 13 ++--- .../test_stateful_cluster_linux/action.yml | 12 +--- .../test_stateful_hive_standalone/action.yml | 19 +----- .../action.yml | 12 +--- .../test_stateful_large_data/action.yml | 13 +---- .../test_stateful_standalone_linux/action.yml | 12 +--- .../test_stateless_cluster_linux/action.yml | 12 +--- .../test_stateless_cluster_macos/action.yml | 12 +--- .../action.yml | 13 +---- .../action.yml | 12 +--- .github/workflows/dev.yml | 10 ---- .github/workflows/reuse.linux.hive.yml | 58 ------------------- .github/workflows/reuse.linux.yml | 57 ++++++------------ Cargo.toml | 4 +- docker/build-tool/alpine/Dockerfile | 6 +- docker/build-tool/debian/Dockerfile | 6 +- scripts/setup/run_build_tool.sh | 2 +- .../5_ee/00_check/00_0014_license_info.py | 2 +- 40 files changed, 113 insertions(+), 427 deletions(-) create mode 100644 .github/actions/setup_test/action.yml delete mode 100644 .github/workflows/reuse.linux.hive.yml diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml index 3e5daae8778e1..fc8eacc23039c 100644 --- a/.github/actions/build_linux/action.yml +++ b/.github/actions/build_linux/action.yml @@ -69,11 +69,12 @@ runs: shell: bash run: | artifacts="meta,metactl,metabench,query,sqllogictests" + binaries="" for artifact in ${artifacts//,/ }; do - echo "==> building databend-$artifact ..." - cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact + binaries="${binaries} --bin databend-$artifact" done - ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact + cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} ${binaries} + ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-* - name: Build Release for specific artifacts if: inputs.artifacts != 'all' @@ -83,8 +84,8 @@ runs: for artifact in ${artifacts//,/ }; do echo "==> building databend-$artifact ..." cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact + ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact done - ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact - name: Check Binary Info shell: bash diff --git a/.github/actions/setup_build_tool/action.yml b/.github/actions/setup_build_tool/action.yml index 8e73ddd76894f..32b7e0e53427a 100644 --- a/.github/actions/setup_build_tool/action.yml +++ b/.github/actions/setup_build_tool/action.yml @@ -38,12 +38,12 @@ runs: EOF RUNNER_PROVIDER="${RUNNER_PROVIDER:-github}" - export SCCACHE_IDLE_TIMEOUT=0 case ${RUNNER_PROVIDER} in aws) echo "setting up sccache for AWS S3..." cat <>$BIN_LOCAL/build-tool export ENABLE_SCCACHE=true + export SCCACHE_IDLE_TIMEOUT=0 export SCCACHE_BUCKET=databend-ci export SCCACHE_REGION=us-east-2 export SCCACHE_S3_KEY_PREFIX="cache/sccache/" @@ -57,6 +57,7 @@ runs: echo "setting up sccache for Google Cloud Storage..." cat <>$BIN_LOCAL/build-tool export ENABLE_SCCACHE=true + export SCCACHE_IDLE_TIMEOUT=0 export SCCACHE_GCS_RW_MODE=READ_WRITE export SCCACHE_GCS_BUCKET=databend-ci export SCCACHE_GCS_KEY_PREFIX="cache/sccache/" diff --git a/.github/actions/setup_test/action.yml b/.github/actions/setup_test/action.yml new file mode 100644 index 0000000000000..5c1f60f8f9f07 --- /dev/null +++ b/.github/actions/setup_test/action.yml @@ -0,0 +1,44 @@ +name: "Setup For GNU Test" +description: "Setup For GNU Test" +inputs: + path: + description: "default to ./target/${BUILD_PROFILE}/" + required: false + default: "" + artifacts: + description: "Artifacts to download, only works with s3/gcs" + required: false + default: "meta,query" +runs: + using: "composite" + steps: + - id: target + shell: bash + run: | + arch=$(uname -m) + target="${arch}-unknown-linux-gnu" + echo "target=${target}" >> $GITHUB_OUTPUT + + - uses: ./.github/actions/setup_bendsql + + - uses: ./.github/actions/artifact_download + with: + sha: ${{ github.sha }} + target: ${{ steps.target.outputs.target }} + category: full + artifacts: ${{ inputs.artifacts }} + path: ${{ inputs.path }} + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "11" + + - name: setup java env + shell: bash + run: | + echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${{ env.LD_LIBRARY_PATH }}" >> $GITHUB_ENV diff --git a/.github/actions/test_cloud_sqllogic_standalone_linux/action.yml b/.github/actions/test_cloud_sqllogic_standalone_linux/action.yml index d1ce97299b175..f76b17406a9d0 100644 --- a/.github/actions/test_cloud_sqllogic_standalone_linux/action.yml +++ b/.github/actions/test_cloud_sqllogic_standalone_linux/action.yml @@ -1,10 +1,6 @@ name: "Test cloud control sqllogic Standalone" description: "Running cloud control sqllogic tests in standalone mode" inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" dirs: description: "logic test suites dirs" required: true @@ -21,11 +17,8 @@ inputs: runs: using: "composite" steps: - - name: Download artifact - uses: ./.github/actions/artifact_download + - uses: ./.github/actions/setup_test with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} artifacts: sqllogictests,meta,query - name: Run sqllogic Tests with Standalone mode diff --git a/.github/actions/test_compat_fuse/action.yml b/.github/actions/test_compat_fuse/action.yml index 854371f0de04a..3e981f465455d 100644 --- a/.github/actions/test_compat_fuse/action.yml +++ b/.github/actions/test_compat_fuse/action.yml @@ -1,19 +1,10 @@ name: "Test fuse-table format in an old query is compatible with new query" description: "Download old binaries and current binaries, write data with old query, read data with new query" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download + - uses: ./.github/actions/setup_test with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} path: ./bins/current artifacts: sqllogictests,meta,query diff --git a/.github/actions/test_compat_meta_meta/action.yml b/.github/actions/test_compat_meta_meta/action.yml index 72c44108af467..38d86b9188e46 100644 --- a/.github/actions/test_compat_meta_meta/action.yml +++ b/.github/actions/test_compat_meta_meta/action.yml @@ -1,18 +1,10 @@ name: "Test protocol compatibility between databend-meta servers" description: "Download old binaries and current binaries, write some data and then read data" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - name: Download artifact - uses: ./.github/actions/artifact_download + - uses: ./.github/actions/setup_test with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} path: ./bins/current/bin artifacts: meta,metactl,metabench diff --git a/.github/actions/test_compat_meta_query/action.yml b/.github/actions/test_compat_meta_query/action.yml index 0945fdfe3248f..0f9bc1e4d0962 100644 --- a/.github/actions/test_compat_meta_query/action.yml +++ b/.github/actions/test_compat_meta_query/action.yml @@ -1,18 +1,10 @@ name: "Test old metasrv is compatible with new query; new meta is compatible with old query" description: "Download old binaries and current binaries, run stateless test with old `meta` and latest `query`" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - name: Download artifact - uses: ./.github/actions/artifact_download + - uses: ./.github/actions/setup_test with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} path: ./bins/current artifacts: sqllogictests,meta,query diff --git a/.github/actions/test_ee_management_mode_linux/action.yml b/.github/actions/test_ee_management_mode_linux/action.yml index a94a4f7c22b72..8c5c9a9ae7323 100644 --- a/.github/actions/test_ee_management_mode_linux/action.yml +++ b/.github/actions/test_ee_management_mode_linux/action.yml @@ -1,19 +1,9 @@ name: "Test ee management mode" description: "Running stateless tests in management mode" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} + - uses: ./.github/actions/setup_test - name: Run Stateful Tests with Standalone mode shell: bash diff --git a/.github/actions/test_ee_sqllogic_standalone_linux/action.yml b/.github/actions/test_ee_sqllogic_standalone_linux/action.yml index 9b5cf31edc3b1..74d8157b7abbc 100644 --- a/.github/actions/test_ee_sqllogic_standalone_linux/action.yml +++ b/.github/actions/test_ee_sqllogic_standalone_linux/action.yml @@ -1,10 +1,6 @@ name: "Test ee sqllogic Standalone" description: "Running ee sqllogic tests in standalone mode" inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" dirs: description: "logic test suites dirs" required: true @@ -21,12 +17,8 @@ inputs: runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download + - uses: ./.github/actions/setup_test with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} artifacts: sqllogictests,meta,query - name: Run sqllogic Tests with Standalone mode diff --git a/.github/actions/test_ee_standalone_background_linux/action.yml b/.github/actions/test_ee_standalone_background_linux/action.yml index 53bd15130cd65..d58b5b66ecc54 100644 --- a/.github/actions/test_ee_standalone_background_linux/action.yml +++ b/.github/actions/test_ee_standalone_background_linux/action.yml @@ -1,19 +1,9 @@ name: "Test Enterprise Background Service features Standalone" description: "Running stateless tests in standalone mode" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} + - uses: ./.github/actions/setup_test - name: Minio Setup for (ubuntu-latest only) shell: bash diff --git a/.github/actions/test_ee_standalone_fake_time_linux/action.yml b/.github/actions/test_ee_standalone_fake_time_linux/action.yml index 545956b97c09b..e8406b2b544e9 100644 --- a/.github/actions/test_ee_standalone_fake_time_linux/action.yml +++ b/.github/actions/test_ee_standalone_fake_time_linux/action.yml @@ -1,19 +1,9 @@ name: "Test Enterprise features Standalone" description: "Running stateless tests in standalone mode" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} + - uses: ./.github/actions/setup_test - name: Minio Setup for (ubuntu-latest only) shell: bash diff --git a/.github/actions/test_ee_standalone_linux/action.yml b/.github/actions/test_ee_standalone_linux/action.yml index f033113b61d0a..6bae9ee1f5794 100644 --- a/.github/actions/test_ee_standalone_linux/action.yml +++ b/.github/actions/test_ee_standalone_linux/action.yml @@ -1,19 +1,9 @@ name: "Test Enterprise features Standalone" description: "Running stateless tests in standalone mode" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} + - uses: ./.github/actions/setup_test - name: Minio Setup for (ubuntu-latest only) shell: bash diff --git a/.github/actions/test_fuzz_standalone_linux/action.yml b/.github/actions/test_fuzz_standalone_linux/action.yml index 855415e85f0d4..6913c54f04fef 100644 --- a/.github/actions/test_fuzz_standalone_linux/action.yml +++ b/.github/actions/test_fuzz_standalone_linux/action.yml @@ -1,20 +1,9 @@ name: "Test Fuzz Standalone" description: "Running fuzz tests in standalone mode" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} + - uses: ./.github/actions/setup_test - name: Test setup shell: bash diff --git a/.github/actions/test_logs/action.yml b/.github/actions/test_logs/action.yml index a65bcf7b9c851..cb5461a2aee60 100644 --- a/.github/actions/test_logs/action.yml +++ b/.github/actions/test_logs/action.yml @@ -1,20 +1,9 @@ name: "Test collection query & profile logs for databend query" description: "Test collection query & profile logs for databend query" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} - artifacts: meta,query + - uses: ./.github/actions/setup_test - name: Run logging Tests shell: bash diff --git a/.github/actions/test_meta_cluster/action.yml b/.github/actions/test_meta_cluster/action.yml index 07248fbde1d80..d956eb28eb9dd 100644 --- a/.github/actions/test_meta_cluster/action.yml +++ b/.github/actions/test_meta_cluster/action.yml @@ -1,18 +1,9 @@ name: "Test databend-meta cluster" description: "Download binaries, setup cluster by joining and leaving" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} + - uses: ./.github/actions/setup_test - name: Test databend-meta cluster shell: bash diff --git a/.github/actions/test_metactl/action.yml b/.github/actions/test_metactl/action.yml index 75c676019fb3d..84b8c68796305 100644 --- a/.github/actions/test_metactl/action.yml +++ b/.github/actions/test_metactl/action.yml @@ -1,18 +1,10 @@ name: "Test metactl" description: "" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - name: Download artifact - uses: ./.github/actions/artifact_download + - uses: ./.github/actions/setup_test with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} artifacts: metactl,meta - name: Test metactl diff --git a/.github/actions/test_sqllogic_cluster_linux/action.yml b/.github/actions/test_sqllogic_cluster_linux/action.yml index df7894e54a4cc..f6ab46b79ac8e 100644 --- a/.github/actions/test_sqllogic_cluster_linux/action.yml +++ b/.github/actions/test_sqllogic_cluster_linux/action.yml @@ -1,10 +1,6 @@ name: "Test sqllogic Cluster Linux" description: "Running sqllogic tests in cluster mode" inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" dirs: description: "logic test suites dirs" required: true @@ -16,12 +12,8 @@ inputs: runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download + - uses: ./.github/actions/setup_test with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} artifacts: sqllogictests,meta,query - name: Run sqllogic Tests with Cluster mode diff --git a/.github/actions/test_sqllogic_iceberg_tpch/action.yml b/.github/actions/test_sqllogic_iceberg_tpch/action.yml index 760d5029acefb..125dd9e30f99b 100644 --- a/.github/actions/test_sqllogic_iceberg_tpch/action.yml +++ b/.github/actions/test_sqllogic_iceberg_tpch/action.yml @@ -1,10 +1,6 @@ name: "Test sqllogic iceberg tpch" description: "Running sqllogic tests in standalone mode" inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" dirs: description: "logic test suites dirs" required: true @@ -16,13 +12,10 @@ inputs: runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download + - uses: ./.github/actions/setup_test with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} artifacts: sqllogictests,meta,query + - name: Iceberg Setup for (ubuntu-latest only) shell: bash run: | diff --git a/.github/actions/test_sqllogic_management_mode_linux/action.yml b/.github/actions/test_sqllogic_management_mode_linux/action.yml index 722aee7571703..d7bae6ce6079b 100644 --- a/.github/actions/test_sqllogic_management_mode_linux/action.yml +++ b/.github/actions/test_sqllogic_management_mode_linux/action.yml @@ -1,10 +1,6 @@ name: "Test sqllogic management mode" description: "Running sqllogic tests in management mode" inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" dirs: description: "logic test suites dirs" required: true @@ -16,12 +12,8 @@ inputs: runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download + - uses: ./.github/actions/setup_test with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} artifacts: sqllogictests,meta,query - name: Run sqllogic Tests with management mode diff --git a/.github/actions/test_sqllogic_stage/action.yml b/.github/actions/test_sqllogic_stage/action.yml index 4627eb9362321..3091e64949a42 100644 --- a/.github/actions/test_sqllogic_stage/action.yml +++ b/.github/actions/test_sqllogic_stage/action.yml @@ -1,10 +1,6 @@ name: "Test sqllogic Stage" description: "Running sqllogic tests in standalone mode" inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" dirs: description: "logic test suites dirs" required: true @@ -20,13 +16,10 @@ inputs: runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download + - uses: ./.github/actions/setup_test with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} artifacts: sqllogictests,meta,query + - name: Minio Setup for (ubuntu-latest only) shell: bash run: | diff --git a/.github/actions/test_sqllogic_standalone_linux/action.yml b/.github/actions/test_sqllogic_standalone_linux/action.yml index 8293d68352530..1aa1f2080523b 100644 --- a/.github/actions/test_sqllogic_standalone_linux/action.yml +++ b/.github/actions/test_sqllogic_standalone_linux/action.yml @@ -1,10 +1,6 @@ name: "Test sqllogic Standalone" description: "Running sqllogic tests in standalone mode" inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" dirs: description: "logic test suites dirs" required: true @@ -24,17 +20,13 @@ inputs: enable_table_meta_cache: description: "Enable table meta cache" required: false - default: true + default: "true" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download + - uses: ./.github/actions/setup_test with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} artifacts: sqllogictests,meta,query - name: Run sqllogic Tests with Standalone mode diff --git a/.github/actions/test_sqllogic_standalone_linux_minio/action.yml b/.github/actions/test_sqllogic_standalone_linux_minio/action.yml index 4a9da48e9e366..1897db5fa2bb7 100644 --- a/.github/actions/test_sqllogic_standalone_linux_minio/action.yml +++ b/.github/actions/test_sqllogic_standalone_linux_minio/action.yml @@ -1,10 +1,6 @@ name: "Test sqllogic Standalone(minio)" description: "Running sqllogic tests in standalone mode" inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" dirs: description: "logic test suites dirs" required: true @@ -21,12 +17,8 @@ inputs: runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download + - uses: ./.github/actions/setup_test with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} artifacts: sqllogictests,meta,query - name: Minio Setup for (ubuntu-latest only) diff --git a/.github/actions/test_sqllogic_standalone_macos/action.yml b/.github/actions/test_sqllogic_standalone_macos/action.yml index a02d1404424e1..36bc985a8aaa7 100644 --- a/.github/actions/test_sqllogic_standalone_macos/action.yml +++ b/.github/actions/test_sqllogic_standalone_macos/action.yml @@ -1,10 +1,6 @@ name: "Test sqllogic Standalone" description: "Running sqllogic tests in standalone mode" inputs: - target: - description: "" - required: true - default: "x86_64-apple-darwin" dirs: description: "logic test suites dirs" required: true @@ -16,23 +12,22 @@ inputs: runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download + - uses: ./.github/actions/setup_test with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} artifacts: sqllogictests,meta,query + - name: Test setup shell: bash run: | bash ./scripts/setup/dev_setup.sh -yd + - name: Run sqllogic Tests with Standalone mode with embedded meta-store shell: bash env: TEST_HANDLERS: ${{ inputs.handlers }} run: | bash ./scripts/ci/ci-run-sqllogic-tests.sh ${{ inputs.dirs }} + - name: Upload failure if: failure() uses: ./.github/actions/artifact_failure diff --git a/.github/actions/test_stateful_cluster_linux/action.yml b/.github/actions/test_stateful_cluster_linux/action.yml index b012d26c4177c..6964d125f7907 100644 --- a/.github/actions/test_stateful_cluster_linux/action.yml +++ b/.github/actions/test_stateful_cluster_linux/action.yml @@ -1,19 +1,9 @@ name: "Test Stateful Cluster Linux" description: "Running stateful tests in cluster mode" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} + - uses: ./.github/actions/setup_test - name: Minio Setup for (ubuntu-latest only) shell: bash diff --git a/.github/actions/test_stateful_hive_standalone/action.yml b/.github/actions/test_stateful_hive_standalone/action.yml index ad8ee0e62e004..13ff2579193aa 100644 --- a/.github/actions/test_stateful_hive_standalone/action.yml +++ b/.github/actions/test_stateful_hive_standalone/action.yml @@ -1,26 +1,9 @@ name: "(hive) Test Stateful Standalone" description: "(hive) Running stateful tests in standalone mode" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} - category: hdfs - - - name: Checkout java env - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: "11" + - uses: ./.github/actions/setup_test # hive cluster setup - name: Hive Cluster Setup diff --git a/.github/actions/test_stateful_iceberg_rest_standalone/action.yml b/.github/actions/test_stateful_iceberg_rest_standalone/action.yml index 14ccbe3f4cedf..5490e0ab00270 100644 --- a/.github/actions/test_stateful_iceberg_rest_standalone/action.yml +++ b/.github/actions/test_stateful_iceberg_rest_standalone/action.yml @@ -1,19 +1,9 @@ name: "Test Stateful Standalone Iceberg Rest" description: "Running Stateful Iceberg Rest tests in standalone mode" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} + - uses: ./.github/actions/setup_test - name: Iceberg Rest Setup for (ubuntu-latest only) shell: bash diff --git a/.github/actions/test_stateful_large_data/action.yml b/.github/actions/test_stateful_large_data/action.yml index 6ff3085984fa9..13348a44b25e3 100644 --- a/.github/actions/test_stateful_large_data/action.yml +++ b/.github/actions/test_stateful_large_data/action.yml @@ -1,20 +1,9 @@ name: "Test Stateful Large Data" description: "Running stateful with large data" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} + - uses: ./.github/actions/setup_test - name: Run Stateful Tests with Standalone mode shell: bash diff --git a/.github/actions/test_stateful_standalone_linux/action.yml b/.github/actions/test_stateful_standalone_linux/action.yml index 211da8f3b73fc..90830bf3abd9a 100644 --- a/.github/actions/test_stateful_standalone_linux/action.yml +++ b/.github/actions/test_stateful_standalone_linux/action.yml @@ -1,19 +1,9 @@ name: "Test Stateful Standalone Linux" description: "Running stateful tests in standalone mode" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} + - uses: ./.github/actions/setup_test - name: Minio Setup for (ubuntu-latest only) shell: bash diff --git a/.github/actions/test_stateless_cluster_linux/action.yml b/.github/actions/test_stateless_cluster_linux/action.yml index b38de4913dae5..98bd2147ad77f 100644 --- a/.github/actions/test_stateless_cluster_linux/action.yml +++ b/.github/actions/test_stateless_cluster_linux/action.yml @@ -1,19 +1,9 @@ name: "Test Stateless Cluster Linux" description: "Running stateless tests in cluster mode" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} + - uses: ./.github/actions/setup_test - name: Test setup shell: bash diff --git a/.github/actions/test_stateless_cluster_macos/action.yml b/.github/actions/test_stateless_cluster_macos/action.yml index 3bebc44f7d558..024106d5299c1 100644 --- a/.github/actions/test_stateless_cluster_macos/action.yml +++ b/.github/actions/test_stateless_cluster_macos/action.yml @@ -1,20 +1,10 @@ name: "Test Stateless Cluster MacOS" description: "Running stateless tests in cluster mode" -inputs: - target: - description: "" - required: true - default: "x86_64-apple-darwin" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql + - uses: ./.github/actions/setup_test - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} - name: Test setup shell: bash run: | diff --git a/.github/actions/test_stateless_standalone_linux/action.yml b/.github/actions/test_stateless_standalone_linux/action.yml index 8df0e11612ab0..09033cb945bf0 100644 --- a/.github/actions/test_stateless_standalone_linux/action.yml +++ b/.github/actions/test_stateless_standalone_linux/action.yml @@ -1,20 +1,9 @@ name: "Test Stateless Standalone" description: "Running stateless tests in standalone mode" -inputs: - target: - description: "" - required: true - default: "x86_64-unknown-linux-gnu" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} + - uses: ./.github/actions/setup_test - name: Test setup shell: bash diff --git a/.github/actions/test_stateless_standalone_macos/action.yml b/.github/actions/test_stateless_standalone_macos/action.yml index 5b8a1d473e1f3..017ac3f6773d5 100644 --- a/.github/actions/test_stateless_standalone_macos/action.yml +++ b/.github/actions/test_stateless_standalone_macos/action.yml @@ -1,19 +1,9 @@ name: "Test Stateless Standalone" description: "Running stateless tests in standalone mode" -inputs: - target: - description: "" - required: true - default: "x86_64-apple-darwin" runs: using: "composite" steps: - - uses: ./.github/actions/setup_bendsql - - name: Download artifact - uses: ./.github/actions/artifact_download - with: - sha: ${{ github.sha }} - target: ${{ inputs.target }} + - uses: ./.github/actions/setup_test - name: Test setup shell: bash diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index c6b6e8289b153..59a627287ffe5 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -55,22 +55,12 @@ jobs: runner_provider: aws license_type: trial - linux_hive: - needs: changes - if: needs.changes.outputs.any_src_changed == 'true' - uses: ./.github/workflows/reuse.linux.hive.yml - secrets: inherit - with: - build_profile: ci - runner_provider: aws - ready: if: always() runs-on: ubuntu-latest needs: - changes - linux - - linux_hive steps: - name: Check Ready to Merge uses: actions/github-script@v7 diff --git a/.github/workflows/reuse.linux.hive.yml b/.github/workflows/reuse.linux.hive.yml deleted file mode 100644 index a19b6675d0c89..0000000000000 --- a/.github/workflows/reuse.linux.hive.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Linux Hive - -on: - workflow_call: - inputs: - build_profile: - description: "Build profile, debug or release" - type: string - required: true - default: "debug" - runner_provider: - description: "Self-hosted runner provider, aws or gcp" - type: string - required: true - default: "aws" - -env: - BUILD_PROFILE: ${{ inputs.build_profile }} - RUNNER_PROVIDER: ${{ inputs.runner_provider }} - -jobs: - build: - runs-on: [self-hosted, X64, Linux, 4c16g, "${{ inputs.runner_provider }}"] - strategy: - matrix: - include: - - { arch: x86_64, libc: gnu } - steps: - - uses: actions/checkout@v4 - with: - # fetch all tags, metasrv and metaclient need tag as its version. - fetch-depth: 0 - - uses: ./.github/actions/build_linux - timeout-minutes: 60 - env: - DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} - with: - sha: ${{ github.sha }} - target: ${{ matrix.arch }}-unknown-linux-${{ matrix.libc }} - features: storage-hdfs - category: hdfs - artifacts: meta,query - - name: upload sccache log - if: always() - uses: actions/upload-artifact@v4 - with: - name: sccache-build-hive - path: target/sccache.log - - test_stateful_hive_standalone: - needs: build - runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/test_stateful_hive_standalone - timeout-minutes: 10 - with: - target: x86_64-unknown-linux-gnu diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml index 6909ac2cbe1f8..dd5a2652a2fda 100644 --- a/.github/workflows/reuse.linux.yml +++ b/.github/workflows/reuse.linux.yml @@ -41,39 +41,7 @@ jobs: - self-hosted - "${{ matrix.runner }}" - Linux - - 8c32g - - "${{ inputs.runner_provider }}" - strategy: - fail-fast: false - matrix: - include: - - { arch: x86_64, runner: X64 } - steps: - - uses: actions/checkout@v4 - with: - # fetch all tags, metasrv and metaclient need tag as its version. - fetch-depth: 0 - - uses: ./.github/actions/build_linux - timeout-minutes: 60 - env: - DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} - with: - sha: ${{ github.sha }} - target: ${{ matrix.arch }}-unknown-linux-gnu - artifacts: all - - name: upload sccache log - if: always() - uses: actions/upload-artifact@v4 - with: - name: sccache-build - path: target/sccache.log - - build_udf: - runs-on: - - self-hosted - - "${{ matrix.runner }}" - - Linux - - 8c32g + - 16c64g - "${{ inputs.runner_provider }}" strategy: fail-fast: false @@ -84,7 +52,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - # fetch all tags, metasrv and metaclient need tag as its version. + # fetch all tags, metasrv and meta client need tag as its version. fetch-depth: 0 - uses: ./.github/actions/build_linux timeout-minutes: 60 @@ -93,14 +61,14 @@ jobs: with: sha: ${{ github.sha }} target: ${{ matrix.arch }}-unknown-linux-gnu - features: python-udf - category: udf - artifacts: query + features: python-udf,storage-hdfs + category: full + artifacts: all - name: upload sccache log if: always() uses: actions/upload-artifact@v4 with: - name: sccache-build-udf + name: sccache-build-${{ matrix.arch }} path: target/sccache.log # build_address_sanitizer: @@ -256,6 +224,19 @@ jobs: with: name: test-stateful-iceberg-rest-standalone + test_stateful_hive_standalone: + needs: [build, check] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/test_stateful_hive_standalone + timeout-minutes: 10 + - name: Upload failure + if: failure() + uses: ./.github/actions/artifact_failure + with: + name: test-stateful-iceberg-rest-standalone + # test_fuzz_standalone: # runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] # needs: [build, check] diff --git a/Cargo.toml b/Cargo.toml index 09c36fdd6fb9b..eb73f1ccd05f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -574,8 +574,8 @@ result_large_err = "allow" ## DONT'T DELETE THIS: If we want best performance, we should use this profile but it will take longer time to compile. ## Test SQL: -## select sum(number) from numbers_mt(10000000000); ~ 3x performance -## select max(number) from numbers_mt(10000000000); ~ 3x performance +## select sum(number) from numbers_mt(10000000000); ~ 3x performance +## select max(number) from numbers_mt(10000000000); ~ 3x performance # [profile.release] # debug = 1 # lto = "thin" diff --git a/docker/build-tool/alpine/Dockerfile b/docker/build-tool/alpine/Dockerfile index 0bebd1f4dda6e..d4c36a0116442 100644 --- a/docker/build-tool/alpine/Dockerfile +++ b/docker/build-tool/alpine/Dockerfile @@ -18,10 +18,8 @@ RUN chmod +x /build/scripts/setup/dev_setup.sh && \ mv /root/.cargo /opt/rust/cargo && \ mv /root/.rustup /opt/rust/rustup -RUN curl -sSfLo /tmp/mold.tar.gz https://github.com/rui314/mold/releases/download/v2.4.0/mold-2.4.0-$(uname -m)-linux.tar.gz && \ - tar --strip-components=1 -xzf /tmp/mold.tar.gz -C /usr/local && \ - rm -rf /tmp/mold.tar.gz && \ - ln -sf /usr/local/bin/mold /usr/bin/$(uname -m)-linux-gnu-ld +RUN curl -sSfLo /tmp/mold.tar.gz https://github.com/rui314/mold/releases/download/v2.35.1/mold-2.35.1-$(uname -m)-linux.tar.gz && \ + tar --strip-components=1 -xzf /tmp/mold.tar.gz -C /usr/local && rm -rf /tmp/mold.tar.gz ENV RUSTUP_HOME /opt/rust/rustup ENV CARGO_HOME /opt/rust/cargo diff --git a/docker/build-tool/debian/Dockerfile b/docker/build-tool/debian/Dockerfile index 24627403d0eee..b984f0f5e6306 100644 --- a/docker/build-tool/debian/Dockerfile +++ b/docker/build-tool/debian/Dockerfile @@ -25,10 +25,8 @@ RUN chmod +x /build/scripts/setup/dev_setup.sh && \ RUN ln -s $(readlink -f `which javac` | sed "s:/bin/javac::") /opt/java -RUN curl -sSfLo /tmp/mold.tar.gz https://github.com/rui314/mold/releases/download/v2.4.0/mold-2.4.0-$(uname -m)-linux.tar.gz && \ - tar --strip-components=1 -xzf /tmp/mold.tar.gz -C /usr/local && \ - rm -rf /tmp/mold.tar.gz && \ - ln -sf /usr/local/bin/mold /usr/bin/$(uname -m)-linux-gnu-ld +RUN curl -sSfLo /tmp/mold.tar.gz https://github.com/rui314/mold/releases/download/v2.35.1/mold-2.35.1-$(uname -m)-linux.tar.gz && \ + tar --strip-components=1 -xzf /tmp/mold.tar.gz -C /usr/local && rm -rf /tmp/mold.tar.gz ENV LIBZ_SYS_STATIC=1 ENV LZMA_API_STATIC=1 diff --git a/scripts/setup/run_build_tool.sh b/scripts/setup/run_build_tool.sh index c1bdec59e5603..3f0c14e4d60a3 100755 --- a/scripts/setup/run_build_tool.sh +++ b/scripts/setup/run_build_tool.sh @@ -73,7 +73,7 @@ if [[ $ENABLE_SCCACHE == "true" ]]; then env | grep -E "^SCCACHE_" >"${CARGO_HOME}/sccache.env" EXTRA_ARGS="${EXTRA_ARGS} --env RUSTC_WRAPPER=sccache --env-file ${CARGO_HOME}/sccache.env" if [[ $COMMAND == "cargo" ]]; then - COMMAND_ARGS="${COMMAND_ARGS} && sccache --show-stats" + COMMAND_ARGS="mold -run ${COMMAND_ARGS} && sccache --show-stats" fi fi diff --git a/tests/suites/5_ee/00_check/00_0014_license_info.py b/tests/suites/5_ee/00_check/00_0014_license_info.py index 6d658540ed664..73c13d801cba4 100755 --- a/tests/suites/5_ee/00_check/00_0014_license_info.py +++ b/tests/suites/5_ee/00_check/00_0014_license_info.py @@ -49,7 +49,7 @@ def get_license(): issue_at = license[0][3] expire_at = license[0][4] features = license[0][6] - now = datetime.utcnow() + now = datetime.now() if now < issue_at or now > expire_at: print(