diff --git a/.github/actions/artifact_download/action.yml b/.github/actions/artifact_download/action.yml index 5c47f8b3531df..57e15a49c5ddb 100644 --- a/.github/actions/artifact_download/action.yml +++ b/.github/actions/artifact_download/action.yml @@ -8,7 +8,7 @@ inputs: description: "" required: true category: - description: "default/hdfs" + description: "default/hdfs/udf" required: false default: default path: @@ -16,7 +16,7 @@ inputs: required: false default: "" artifacts: - description: "Artifacts to download, only works with s3" + description: "Artifacts to download, only works with s3/gcs" required: false default: "meta,query" outputs: diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml index 68bb2a22b72e8..63228c9a10272 100644 --- a/.github/actions/build_linux/action.yml +++ b/.github/actions/build_linux/action.yml @@ -30,7 +30,7 @@ runs: uses: ./.github/actions/setup_build_tool with: target: ${{ inputs.target }} - bypass_env_vars: RUSTFLAGS,RUST_LOG,PYO3_PYTHON,LD_LIBRARY_PATH,PYO3_CROSS_PYTHON_VERSION + bypass_env_vars: RUSTFLAGS,RUST_LOG - name: Cross setup if: startsWith(inputs.target, 'aarch64-') @@ -61,9 +61,6 @@ runs: echo "RUSTFLAGS=${flags} -C link-arg=-Wl,--compress-debug-sections=zlib" >> $GITHUB_ENV target=${{ inputs.target }} echo "BUILD_ARCH=${target/-unknown-linux-*}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=/usr/local/lib" >> $GITHUB_ENV - echo "PYO3_PYTHON=python3.12" >> $GITHUB_ENV - echo "PYO3_CROSS_PYTHON_VERSION=/3.12" >> $GITHUB_ENV # build all binaries for debug - name: Build Debug @@ -113,10 +110,9 @@ runs: - name: Check Binary Info shell: bash - continue-on-error: true run: | - readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query - ldd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query + readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true + ldd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true # - name: Compress Binaries with UPX # if: env.BUILD_PROFILE == 'debug' diff --git a/.github/actions/build_linux_sanitizer/action.yml b/.github/actions/build_linux_sanitizer/action.yml index 87eb8c2a314a4..1097d470b4752 100644 --- a/.github/actions/build_linux_sanitizer/action.yml +++ b/.github/actions/build_linux_sanitizer/action.yml @@ -19,7 +19,7 @@ runs: uses: ./.github/actions/setup_build_tool with: target: ${{ inputs.target }} - bypass_env_vars: RUSTFLAGS,RUST_LOG,PYO3_PYTHON,LD_LIBRARY_PATH + bypass_env_vars: RUSTFLAGS,RUST_LOG - name: Cross setup if: startsWith(inputs.target, 'aarch64-') @@ -56,11 +56,3 @@ runs: cargo -Zbuild-std -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --manifest-path src/binaries/Cargo.toml --bin databend-$artifact done ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-* - - - name: Check Binary Info - if: env.BUILD_PROFILE == 'debug' && endsWith(inputs.target, '-gnu') - shell: bash - continue-on-error: true - run: | - readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query - ldd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query diff --git a/.github/actions/check/action.yml b/.github/actions/check/action.yml index dd64ec5787091..161e844958e70 100644 --- a/.github/actions/check/action.yml +++ b/.github/actions/check/action.yml @@ -10,7 +10,7 @@ runs: - name: Setup Build Tool uses: ./.github/actions/setup_build_tool with: - bypass_env_vars: RUSTFLAGS,RUST_LOG,GITHUB_TOKEN,PYO3_PYTHON,LD_LIBRARY_PATH + bypass_env_vars: RUSTFLAGS,RUST_LOG,GITHUB_TOKEN - name: Check Apache License Header uses: korandoru/hawkeye@v2 diff --git a/.github/actions/publish_binary/action.yml b/.github/actions/publish_binary/action.yml index 3a3c618a75ac2..52515adb79ae7 100644 --- a/.github/actions/publish_binary/action.yml +++ b/.github/actions/publish_binary/action.yml @@ -8,7 +8,7 @@ inputs: description: "Release target" required: true category: - description: "Release default/hdfs/testsuite" + description: "Release default/hdfs/udf/testsuite" required: false default: default @@ -26,6 +26,9 @@ runs: hdfs) publish_name="databend-hdfs-${{ inputs.version }}-${{ inputs.target }}" ;; + udf ) + publish_name="databend-udf-${{ inputs.version }}-${{ inputs.target }}" + ;; testsuite) publish_name="databend-testsuite-${{ inputs.version }}-${{ inputs.target }}" ;; diff --git a/.github/actions/setup_build_tool/action.yml b/.github/actions/setup_build_tool/action.yml index 290c35a213988..580be26142e6a 100644 --- a/.github/actions/setup_build_tool/action.yml +++ b/.github/actions/setup_build_tool/action.yml @@ -8,7 +8,7 @@ inputs: bypass_env_vars: description: "Environment variables bypass to docker container" required: false - default: RUSTFLAGS,RUST_LOG,PYO3_PYTHON,LD_LIBRARY_PATH + default: RUSTFLAGS,RUST_LOG runs: using: "composite" steps: @@ -28,9 +28,6 @@ runs: cat <$BIN_LOCAL/build-tool #!/bin/bash script_name=\$(basename "\$0") - export PYO3_PYTHON=python3.12 - export LD_LIBRARY_PATH=/usr/local/lib - export PYO3_CROSS_PYTHON_VERSION=3.12 export INTERACTIVE=false export TARGET=${{ inputs.target }} export CARGO_INCREMENTAL=0 diff --git a/.github/actions/test_unit/action.yml b/.github/actions/test_unit/action.yml index 75860bd6a3740..adb8fc5719b44 100644 --- a/.github/actions/test_unit/action.yml +++ b/.github/actions/test_unit/action.yml @@ -7,7 +7,7 @@ runs: - name: Setup Build Tool uses: ./.github/actions/setup_build_tool with: - bypass_env_vars: RUSTFLAGS,RUSTDOCFLAGS,RUST_TEST_THREADS,RUST_LOG,RUST_BACKTRACE,PYO3_PYTHON,LD_LIBRARY_PATH + bypass_env_vars: RUSTFLAGS,RUSTDOCFLAGS,RUST_TEST_THREADS,RUST_LOG,RUST_BACKTRACE - shell: bash run: | diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml index ae30eb428f0c8..d69e055c4c942 100644 --- a/.github/workflows/cancel.yml +++ b/.github/workflows/cancel.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Cancel workflow runs for the PR - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: github-token: ${{ github.token }} script: | diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 49b431e9d7ba4..313bca5f4ba8f 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -71,7 +71,7 @@ jobs: - linux_hive steps: - name: Check Ready to Merge - uses: actions/github-script@v6 + uses: actions/github-script@v7 env: SRC_CHANGED: ${{ needs.changes.outputs.any_src_changed }} LINUX_BUILD_RESULT: ${{ needs.linux.result }} diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml index e693eba00014b..6431f3d2e49b4 100644 --- a/.github/workflows/production.yml +++ b/.github/workflows/production.yml @@ -61,7 +61,7 @@ jobs: - linux steps: - name: Check Ready to Merge - uses: actions/github-script@v6 + uses: actions/github-script@v7 env: SRC_CHANGED: ${{ needs.changes.outputs.any_src_changed }} LINUX_BUILD_RESULT: ${{ needs.linux.result }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cdbd31fc56db3..1188193006ce8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,7 +26,6 @@ env: jobs: create_release: - name: create release runs-on: ubuntu-latest outputs: version: ${{ steps.bump.outputs.tag }} @@ -149,7 +148,7 @@ jobs: branch-suffix: random delete-branch: true - linux: + build_default: runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, gcp] needs: create_release env: @@ -158,23 +157,107 @@ jobs: fail-fast: false matrix: include: - - {target: x86_64-unknown-linux-gnu, runner: X64} - - {target: aarch64-unknown-linux-gnu, runner: ARM64} - - {target: x86_64-unknown-linux-musl, runner: X64} - - {target: aarch64-unknown-linux-musl, runner: X64} + - { target: x86_64-unknown-linux-gnu, runner: X64 } + - { target: aarch64-unknown-linux-gnu, runner: ARM64 } + - { target: x86_64-unknown-linux-musl, runner: X64 } + - { target: aarch64-unknown-linux-musl, runner: X64 } + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Build Release + uses: ./.github/actions/build_linux + with: + sha: ${{ github.sha }} + target: ${{ matrix.target }} + artifacts: sqllogictests,sqlsmith,metactl,meta,query + + build_udf: + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, gcp] + needs: create_release + env: + RUNNER_PROVIDER: gcp + strategy: + fail-fast: false + matrix: + include: + - { target: x86_64-unknown-linux-gnu, runner: X64 } + - { target: aarch64-unknown-linux-gnu, runner: ARM64 } + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Build Release + uses: ./.github/actions/build_linux + with: + sha: ${{ github.sha }} + target: ${{ matrix.target }} + artifacts: sqllogictests,sqlsmith,metactl,meta,query + features: python-udf + category: udf + + build_hdfs: + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, gcp] + needs: create_release + env: + RUNNER_PROVIDER: gcp + strategy: + fail-fast: false + matrix: + include: + - { target: x86_64-unknown-linux-gnu, runner: X64 } + - { target: aarch64-unknown-linux-gnu, runner: ARM64 } steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ needs.create_release.outputs.version }} - name: Build Release uses: ./.github/actions/build_linux with: sha: ${{ github.sha }} target: ${{ matrix.target }} artifacts: sqllogictests,sqlsmith,metactl,meta,query - upload: false + features: storage-hdfs + category: hdfs + + publish: + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 4c8g, gcp] + needs: [create_release, build_default, build_hdfs] + env: + RUNNER_PROVIDER: gcp + strategy: + fail-fast: false + matrix: + include: + - category: default + target: x86_64-unknown-linux-gnu + runner: X64 + - category: default + target: aarch64-unknown-linux-gnu + runner: ARM64 + - category: default + target: x86_64-unknown-linux-musl + runner: X64 + - category: default + target: aarch64-unknown-linux-musl + runner: X64 + - category: hdfs + target: x86_64-unknown-linux-gnu + runner: X64 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ needs.create_release.outputs.version }} + - name: Download artifact + uses: ./.github/actions/artifact_download + with: + sha: ${{ github.sha }} + target: ${{ matrix.target }} + category: ${{ matrix.category }} - name: Basic Sqllogic Test if: matrix.target != 'aarch64-unknown-linux-musl' shell: bash @@ -182,12 +265,19 @@ jobs: TEST_HANDLERS: http run: | mkdir -p target/release - cp ./target/${{ matrix.target }}/release/databend-{meta,query,sqllogictests} ./target/release/ bash ./scripts/ci/ci-run-sqllogic-tests.sh base - name: Pack Binaries run: | target=${{ matrix.target }} version=${{ needs.create_release.outputs.version }} + case ${{ matrix.category }} in + default) + pkg_name="databend-${version}-${target}" + ;; + *) + pkg_name="databend-${{ matrix.category }}-${version}-${target}" + ;; + esac mkdir -p release/${target}/{bin,configs,systemd,scripts} cp ./target/${target}/release/databend-{query,meta,metactl} release/${target}/bin/ rm -f release/${target}/bin/*.d @@ -196,22 +286,12 @@ jobs: cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/ cp -r ./scripts/distribution/package-scripts/* release/${target}/scripts/ - tar -C ./release/${target} -czvf databend-${version}-${target}.tar.gz bin configs systemd scripts readme.txt - sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt - - name: Pack Testsuite - if: matrix.target == 'x86_64-unknown-linux-gnu' - run: | - target=${{ matrix.target }} - version=${{ needs.create_release.outputs.version }} - mkdir -p release/testsuite/bin - cp -r ./tests/sqllogictests/suites ./release/testsuite/ - cp ./target/${target}/release/databend-{sqllogictests,sqlsmith} release/testsuite/bin/ - tar -C ./release/testsuite -czvf databend-testsuite-${version}-${target}.tar.gz bin suites - sha256sum databend-testsuite-${version}-${target}.tar.gz >> sha256-testsuite-${version}-${target}.txt + tar -C ./release/${target} -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt + sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt - name: post sha256 uses: actions/upload-artifact@v4 with: - name: sha256sums-${{ matrix.target }} + name: sha256sums-${{ matrix.category }}-${{ matrix.target }} path: sha256-*.txt retention-days: 1 - name: Publish Binaries @@ -225,103 +305,78 @@ jobs: with: version: ${{ needs.create_release.outputs.version }} target: ${{ matrix.target }} - - name: Publish Testsuite - if: matrix.target == 'x86_64-unknown-linux-gnu' - uses: ./.github/actions/publish_binary - env: - GH_TOKEN: ${{ github.token }} - with: - version: ${{ needs.create_release.outputs.version }} - target: ${{ matrix.target }} - category: testsuite + category: ${{ matrix.category }} - hdfs: - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, gcp] - needs: create_release + publish_testsuite: + runs-on: [self-hosted, X64, Linux, 4c8g, gcp] + needs: [create_release, build_default] env: RUNNER_PROVIDER: gcp strategy: fail-fast: false matrix: - include: - - {target: x86_64-unknown-linux-gnu, runner: X64} + target: + - x86_64-unknown-linux-gnu steps: - name: Checkout uses: actions/checkout@v4 with: - fetch-depth: 0 ref: ${{ needs.create_release.outputs.version }} - - name: Setup Build Tool - uses: ./.github/actions/setup_build_tool - with: - target: ${{ matrix.target }} - - name: Build Release - uses: ./.github/actions/build_linux + - name: Download artifact + uses: ./.github/actions/artifact_download with: sha: ${{ github.sha }} target: ${{ matrix.target }} - artifacts: meta,query - category: hdfs - features: storage-hdfs - upload: false - - name: Pack binaries + category: default + - name: Pack Testsuite run: | target=${{ matrix.target }} version=${{ needs.create_release.outputs.version }} - mkdir -p release/${target}/{bin,configs,systemd,scripts} - cp ./target/${target}/release/databend-* release/${target}/bin/ - rm -f release/${target}/bin/*.d - cp ./scripts/distribution/systemd/databend-* release/${target}/systemd/ - cp ./scripts/distribution/configs/databend-* release/${target}/configs/ - cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt - cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/ - cp -r ./scripts/distribution/package-scripts/* release/${target}/scripts/ - tar -C ./release/${target} -czvf databend-hdfs-${version}-${target}.tar.gz bin configs systemd scripts readme.txt - sha256sum databend-hdfs-${version}-${target}.tar.gz >> sha256-hdfs-${version}-${target}.txt + pkg_name="databend-testsuite-${version}-${target}" + mkdir -p release/testsuite/bin + cp -r ./tests/sqllogictests/suites ./release/testsuite/ + cp ./target/${target}/release/databend-{sqllogictests,sqlsmith} release/testsuite/bin/ + tar -C ./release/testsuite -czvf ${pkg_name}.tar.gz bin suites + sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt - name: post sha256 uses: actions/upload-artifact@v4 with: - name: sha256sums-hdfs-${{ matrix.target }} + name: sha256sums-testsuite-${{ matrix.target }} path: sha256-*.txt retention-days: 1 - - name: Publish Binaries + - name: Publish Testsuite uses: ./.github/actions/publish_binary env: GH_TOKEN: ${{ github.token }} - AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: auto - AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }} with: version: ${{ needs.create_release.outputs.version }} target: ${{ matrix.target }} - category: hdfs + category: testsuite - docker_combined: - name: docker combined - runs-on: ubuntu-latest - needs: [create_release, linux] + docker_all_in_one: + runs-on: [self-hosted, X64, Linux, 4c8g, gcp] + needs: [create_release, build_default] steps: - name: Checkout uses: actions/checkout@v4 with: ref: ${{ needs.create_release.outputs.version }} - - name: Download binaries for usage - id: download_binaries - env: - GH_TOKEN: ${{ github.token }} - run: | - version="${{ needs.create_release.outputs.version }}" - declare -A platform_targets=( ["arm64"]="aarch64-unknown-linux-gnu" ["amd64"]="x86_64-unknown-linux-gnu") - mkdir -p ./distro/ - for platform in ${!platform_targets[@]}; do - target=${platform_targets[$platform]} - gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/ - mkdir -p ./target/${target}/release - tar x -C ./target/${target}/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/ - mkdir -p ./distro/linux/${platform} - cp ./target/${target}/release/databend-* ./distro/linux/${platform} - done + - name: Download artifacts for x86_64 + uses: ./.github/actions/artifact_download + with: + sha: ${{ github.sha }} + target: x86_64-unknown-linux-gnu + category: udf + artifacts: metactl,meta,query + path: distro/linux/amd64 + - name: Download artifacts for aarch64 + uses: ./.github/actions/artifact_download + with: + sha: ${{ github.sha }} + target: aarch64-unknown-linux-gnu + category: udf + artifacts: metactl,meta,query + path: distro/linux/arm64 - name: Set up QEMU uses: docker/setup-qemu-action@v3 - uses: ./.github/actions/setup_docker @@ -373,10 +428,9 @@ jobs: short-description: "A modern cloud data warehouse. Also available in the cloud: https://app.databend.com." readme-filepath: ./docker/README.md - docker_separate: - name: docker separate - runs-on: ubuntu-latest - needs: [create_release, linux] + docker_service: + runs-on: [self-hosted, X64, Linux, 4c8g, gcp] + needs: [create_release, build_udf] strategy: fail-fast: false matrix: @@ -391,22 +445,22 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ needs.create_release.outputs.version }} - - name: Download binaries for usage - id: download_binaries - env: - GH_TOKEN: ${{ github.token }} - run: | - version="${{ needs.create_release.outputs.version }}" - declare -A platform_targets=( ["arm64"]="aarch64-unknown-linux-gnu" ["amd64"]="x86_64-unknown-linux-gnu") - mkdir -p ./distro/ - for platform in ${!platform_targets[@]}; do - target=${platform_targets[$platform]} - gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/ - mkdir -p ./target/${target}/release - tar x -C ./target/${target}/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/ - mkdir -p ./distro/linux/${platform} - cp ./target/${target}/release/databend-* ./distro/linux/${platform} - done + - name: Download artifacts for x86_64 + uses: ./.github/actions/artifact_download + with: + sha: ${{ github.sha }} + target: x86_64-unknown-linux-gnu + category: udf + artifacts: ${{ matrix.service }} + path: distro/linux/amd64 + - name: Download artifacts for aarch64 + uses: ./.github/actions/artifact_download + with: + sha: ${{ github.sha }} + target: aarch64-unknown-linux-gnu + category: udf + artifacts: ${{ matrix.service }} + path: distro/linux/arm64 - name: Set up QEMU uses: docker/setup-qemu-action@v3 - uses: ./.github/actions/setup_docker @@ -454,8 +508,8 @@ jobs: file: ./docker/${{ matrix.distro }}/${{ matrix.service }}.Dockerfile distribution: - runs-on: ubuntu-latest - needs: [create_release, linux] + runs-on: [self-hosted, X64, Linux, 4c8g, gcp] + needs: [create_release, build_default] strategy: matrix: arch: @@ -479,16 +533,14 @@ jobs: id: target run: | echo 'target=${{ matrix.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT - - name: Download binaries for usage - id: download_binaries - env: - GH_TOKEN: ${{ github.token }} - run: | - target=${{ steps.target.outputs.target }} - version="${{ needs.create_release.outputs.version }}" - mkdir -p ./distro/ - gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/ - tar x -C distro -f ./distro/databend-${version}-${target}.tar.gz + - name: Download artifacts + uses: ./.github/actions/artifact_download + with: + sha: ${{ github.sha }} + target: ${{ steps.target.outputs.target }} + category: udf + artifacts: metactl,meta,query + path: distro - name: Build Packages id: build_packages run: | @@ -534,7 +586,7 @@ jobs: gpg_signing_key: ${{ secrets.GPG_KEY_DEB }} sha256sums: - needs: [create_release, linux, distribution] + needs: [create_release, publish, distribution] runs-on: ubuntu-latest steps: - name: checkout @@ -561,7 +613,7 @@ jobs: gh release upload ${version} sha256sums.txt --clobber benchmark: - needs: [create_release, docker_separate] + needs: [create_release, docker_service] uses: ./.github/workflows/reuse.benchmark.yml secrets: inherit with: @@ -574,25 +626,20 @@ jobs: target: all sqlsmith: - needs: [create_release, linux] - runs-on: ubuntu-latest + needs: [create_release, build_default] + runs-on: [self-hosted, X64, Linux, 4c8g, gcp] steps: - uses: actions/checkout@v4 with: ref: ${{ needs.create_release.outputs.version }} - - name: Download artifact for release - env: - GH_TOKEN: ${{ github.token }} - run: | - version=${{ needs.create_release.outputs.version }} - target=x86_64-unknown-linux-gnu - mkdir -p ./distro/ - mkdir -p ./target/release/ - gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/ - gh release download ${version} --pattern "databend-testsuite-${version}-${target}.tar.gz" --dir distro/ - tar x -C ./target/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/ - tar x -C ./target/release -f ./distro/databend-testsuite-${version}-${target}.tar.gz --strip-components 1 bin/ - chmod +x ./target/release/databend-* + - name: Download artifacts + uses: ./.github/actions/artifact_download + with: + sha: ${{ github.sha }} + target: x86_64-unknown-linux-gnu + category: udf + artifacts: meta,query,sqlsmith + path: distro - name: Run sqlsmith timeout-minutes: 60 shell: bash @@ -617,9 +664,9 @@ jobs: if: always() needs: - create_release - - linux - - docker_combined - - docker_separate + - publish + - docker_all_in_one + - docker_service - distribution steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml index 161f385415153..93b2a307e977a 100644 --- a/.github/workflows/reuse.linux.yml +++ b/.github/workflows/reuse.linux.yml @@ -4,15 +4,15 @@ on: workflow_call: inputs: build_profile: - description: 'Build profile, debug or release' + description: "Build profile, debug or release" type: string required: true - default: 'debug' + default: "debug" runner_provider: - description: 'Self-hosted runner provider, aws or gcp' + description: "Self-hosted runner provider, aws or gcp" type: string required: true - default: 'aws' + default: "aws" env: BUILD_PROFILE: ${{ inputs.build_profile }} @@ -54,7 +54,12 @@ jobs: cat license.json build: - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, "${{ inputs.runner_provider }}"] + runs-on: + - self-hosted + - "${{ matrix.runner }}" + - Linux + - 16c32g + - "${{ inputs.runner_provider }}" strategy: fail-fast: false matrix: @@ -72,8 +77,40 @@ jobs: target: ${{ matrix.arch }}-unknown-linux-gnu artifacts: all + build_udf: + runs-on: + - self-hosted + - "${{ matrix.runner }}" + - Linux + - 16c32g + - "${{ inputs.runner_provider }}" + strategy: + fail-fast: false + matrix: + include: + - { arch: x86_64, runner: X64 } + - { arch: aarch64, runner: ARM64 } + steps: + - uses: actions/checkout@v4 + with: + # fetch all tags, metasrv and metaclient need tag as its version. + fetch-depth: 0 + - uses: ./.github/actions/build_linux + timeout-minutes: 60 + with: + sha: ${{ github.sha }} + target: ${{ matrix.arch }}-unknown-linux-gnu + features: python-udf + category: udf + artifacts: query + build_address_sanitizer: - runs-on: [ self-hosted, "${{ matrix.runner }}", Linux, 16c32g, "${{ inputs.runner_provider }}" ] + runs-on: + - self-hosted + - "${{ matrix.runner }}" + - Linux + - 16c32g + - "${{ inputs.runner_provider }}" strategy: fail-fast: false matrix: diff --git a/Makefile b/Makefile index fbf18a5d3353c..befe77b6cc62c 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ sqllogic-test: build stateless-cluster-test: build rm -rf ./_meta*/ - ulimit -n 10000;ulimit -s 16384; bash ./scripts/ci/ci-run-stateless-tests-cluster.sh + ulimit -n 10000; ulimit -s 16384; bash ./scripts/ci/ci-run-stateless-tests-cluster.sh stateless-cluster-test-tls: build rm -rf ./_meta*/ diff --git a/docker/debian/query.Dockerfile b/docker/debian/query.Dockerfile index 28f49f2e22636..ae02043e24881 100644 --- a/docker/debian/query.Dockerfile +++ b/docker/debian/query.Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bookworm +FROM python:3.12-slim-bookworm ARG TARGETPLATFORM ENV TERM=dumb