diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml index 3465052be8c17..502dfdedc47b8 100644 --- a/.github/actions/build_linux/action.yml +++ b/.github/actions/build_linux/action.yml @@ -27,13 +27,6 @@ runs: using: "composite" steps: - name: Setup Build Tool - if: endsWith(inputs.target, '-linux-gnu') - uses: ./.github/actions/setup_build_tool - with: - target: base - bypass_env_vars: RUSTFLAGS,RUST_LOG - - name: Setup Build Tool - if: endsWith(inputs.target, '-linux-musl') uses: ./.github/actions/setup_build_tool with: target: ${{ inputs.target }} @@ -53,13 +46,13 @@ runs: flags="-C target-feature=+sse4.2" ;; x86_64-unknown-linux-musl) - flags="-C target-feature=+sse4.2" + flags="-C target-feature=+sse4.2 -C link-arg=--max-memory=4294967296" ;; aarch64-unknown-linux-gnu) flags="" ;; aarch64-unknown-linux-musl) - flags="" + flags="-C link-arg=--max-memory=4294967296" ;; *) flags="" diff --git a/.github/actions/build_linux_sanitizer/action.yml b/.github/actions/build_linux_sanitizer/action.yml index b24cd1a1c3391..a13868d2d0566 100644 --- a/.github/actions/build_linux_sanitizer/action.yml +++ b/.github/actions/build_linux_sanitizer/action.yml @@ -16,13 +16,6 @@ runs: using: "composite" steps: - name: Setup Build Tool - if: endsWith(inputs.target, '-linux-gnu') - uses: ./.github/actions/setup_build_tool - with: - target: base - bypass_env_vars: RUSTFLAGS,RUST_LOG - - name: Setup Build Tool - if: endsWith(inputs.target, '-linux-musl') uses: ./.github/actions/setup_build_tool with: target: ${{ inputs.target }} diff --git a/.github/actions/setup_docker/action.yml b/.github/actions/setup_docker/action.yml index 81e3cbc75d5a8..bd1b6de42dfad 100644 --- a/.github/actions/setup_docker/action.yml +++ b/.github/actions/setup_docker/action.yml @@ -25,7 +25,7 @@ runs: using: "composite" steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Configure AWS Credentials if: inputs.ecr_role_arn diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml index 563ae94d2a9b2..a3dd18f852940 100644 --- a/.github/workflows/reuse.linux.yml +++ b/.github/workflows/reuse.linux.yml @@ -54,12 +54,12 @@ jobs: cat license.json build: - runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: include: - - { arch: x86_64, libc: gnu } + - { target: x86_64-unknown-linux-gnu, runner: X64 } steps: - uses: actions/checkout@v4 with: @@ -69,16 +69,16 @@ jobs: timeout-minutes: 60 with: sha: ${{ github.sha }} - target: ${{ matrix.arch }}-unknown-linux-${{ matrix.libc }} + target: ${{ matrix.target }} artifacts: all build_address_sanitizer: - runs-on: [ self-hosted, ARM64, Linux, 8c16g, "${{ inputs.runner_provider }}" ] + runs-on: [ self-hosted, "${{ matrix.runner }}", Linux, 16c32g, "${{ inputs.runner_provider }}" ] strategy: fail-fast: false matrix: include: - - { arch: aarch64, libc: gnu } + - { target: aarch64-unknown-linux-gnu, runner: ARM64 } steps: - uses: actions/checkout@v4 with: @@ -87,27 +87,27 @@ jobs: - uses: ./.github/actions/build_linux_sanitizer timeout-minutes: 60 with: - target: ${{ matrix.arch }}-unknown-linux-${{ matrix.libc }} + target: ${{ matrix.target }} artifacts: query - build_other: - runs-on: [self-hosted, X64, Linux, 8c16g, "${{ inputs.runner_provider }}"] - strategy: - fail-fast: false - matrix: - include: - - { arch: aarch64, libc: musl } - steps: - - uses: actions/checkout@v4 - with: - # fetch all tags, metasrv and metaclient need tag as its version. - fetch-depth: 0 - - uses: ./.github/actions/build_linux - timeout-minutes: 60 - with: - sha: ${{ github.sha }} - target: ${{ matrix.arch }}-unknown-linux-${{ matrix.libc }} - artifacts: query + #build_other: + # runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, "${{ inputs.runner_provider }}"] + #strategy: + #fail-fast: false + #matrix: + #include: + #- { target: aarch64-unknown-linux-musl, runner: ARM64 } + #steps: + # - uses: actions/checkout@v4 + #with: + # #fetch all tags, metasrv and metaclient need tag as its version. + #fetch-depth: 0 + # - uses: ./.github/actions/build_linux + #timeout-minutes: 60 + #with: + #sha: ${{ github.sha }} + #target: ${{ matrix.target }} + #artifacts: query test_unit: runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"] diff --git a/Cargo.toml b/Cargo.toml index d824a5bfefba0..837615ff11716 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -261,4 +261,4 @@ sentry = { git = "https://github.com/getsentry/sentry-rust", rev = "6ef6d97" } micromarshal = { git = "https://github.com/ariesdevil/opensrv", rev = "6c96813" } async-backtrace = { git = "https://github.com/zhang2014/async-backtrace.git", rev = "dea4553" } geozero = { git = "https://github.com/georust/geozero", rev = "1d78b36" } -#proj = { git = "https://github.com/ariesdevil/proj", rev = "51e1c60" } +# proj = { git = "https://github.com/ariesdevil/proj", rev = "51e1c60" } diff --git a/docker/build-tool/alpine/Dockerfile b/docker/build-tool/alpine/Dockerfile index ad2234ca763b2..0bebd1f4dda6e 100644 --- a/docker/build-tool/alpine/Dockerfile +++ b/docker/build-tool/alpine/Dockerfile @@ -23,7 +23,6 @@ RUN curl -sSfLo /tmp/mold.tar.gz https://github.com/rui314/mold/releases/downloa rm -rf /tmp/mold.tar.gz && \ ln -sf /usr/local/bin/mold /usr/bin/$(uname -m)-linux-gnu-ld -ENV LIBCLANG_PATH /usr/lib/llvm17/lib ENV RUSTUP_HOME /opt/rust/rustup ENV CARGO_HOME /opt/rust/cargo ENV PATH /opt/rust/cargo/bin:/opt/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/docker/build-tool/dev/Dockerfile b/docker/build-tool/dev/Dockerfile index e5e80ff269680..d05268bdaf542 100644 --- a/docker/build-tool/dev/Dockerfile +++ b/docker/build-tool/dev/Dockerfile @@ -1,8 +1,7 @@ ARG RUST_TOOLCHAIN -FROM datafuselabs/build-tool:base-${RUST_TOOLCHAIN} +FROM datafuselabs/build-tool:debian-${RUST_TOOLCHAIN} -ARG ARCH COPY rust-tools.txt /build/scripts/setup/rust-tools.txt RUN /build/scripts/setup/dev_setup.sh -ycd && \ rm -rf /opt/rust/cargo/git && \ diff --git a/scripts/setup/run_build_tool.sh b/scripts/setup/run_build_tool.sh index b2580fadf6d29..9c9aee8852a34 100755 --- a/scripts/setup/run_build_tool.sh +++ b/scripts/setup/run_build_tool.sh @@ -19,12 +19,29 @@ TOOLCHAIN_VERSION=$(awk -F'[ ="]+' '$1 == "channel" { print $2 }' rust-toolchain _UID=$(id -u) _GID=$(id -g) +USER=${USER:-$(whoami)} + +case $TARGET in +x86_64-unknown-linux-gnu | aarch64-unknown-linux-gnu) + RUNNER_BASE="debian" + USER_CMD="useradd -u ${_UID} -M -s /bin/bash ${USER}" + ;; +x86_64-unknown-linux-musl | aarch64-unknown-linux-musl) + RUNNER_BASE="alpine" + USER_CMD="adduser -u ${_UID} -D -H -s /bin/bash ${USER}" + ;; +*) + RUNNER_BASE="dev" + TARGET=x86_64-unknown-linux-gnu + USER_CMD="useradd -u ${_UID} -M -s /bin/bash ${USER}" + ;; +esac + # skip building temporary image for root if [[ ${_UID} == 0 ]]; then - IMAGE="datafuselabs/build-tool:${TARGET}-${TOOLCHAIN_VERSION}" + IMAGE="datafuselabs/build-tool:${RUNNER_BASE}-${TOOLCHAIN_VERSION}" else - USER=${USER:-$(whoami)} - IMAGE="${USER}/build-tool:${TARGET}-${TOOLCHAIN_VERSION}" + IMAGE="${USER}/build-tool:${RUNNER_BASE}-${TOOLCHAIN_VERSION}" if [[ $(docker image ls "${IMAGE}" --format="true") ]]; then echo "==> build-tool using image ${IMAGE}" @@ -32,8 +49,9 @@ else echo "==> preparing temporary build-tool image ${IMAGE} ..." tmpdir=$(mktemp -d) cat >"${tmpdir}/Dockerfile" < /etc/sudoers.d/databend EOF docker build -t "${IMAGE}" "${tmpdir}"