chore(ci): change build runner with target

.github/actions/build_linux/action.yml

@@ -27,13 +27,6 @@ runs:
   using: "composite"
   steps:
     - name: Setup Build Tool
-      if: endsWith(inputs.target, '-linux-gnu')
-      uses: ./.github/actions/setup_build_tool
-      with:
-        target: base
-        bypass_env_vars: RUSTFLAGS,RUST_LOG
-    - name: Setup Build Tool
-      if: endsWith(inputs.target, '-linux-musl')
       uses: ./.github/actions/setup_build_tool
       with:
         target: ${{ inputs.target }}
@@ -53,13 +46,13 @@ runs:
             flags="-C target-feature=+sse4.2"
             ;;
           x86_64-unknown-linux-musl)
-            flags="-C target-feature=+sse4.2"
+            flags="-C target-feature=+sse4.2 -C link-arg=--max-memory=4294967296"
             ;;
           aarch64-unknown-linux-gnu)
             flags=""
             ;;
           aarch64-unknown-linux-musl)
-            flags=""
+            flags="-C link-arg=--max-memory=4294967296"
             ;;
           *)
             flags=""

.github/actions/build_linux_sanitizer/action.yml

@@ -16,13 +16,6 @@ runs:
   using: "composite"
   steps:
     - name: Setup Build Tool
-      if: endsWith(inputs.target, '-linux-gnu')
-      uses: ./.github/actions/setup_build_tool
-      with:
-        target: base
-        bypass_env_vars: RUSTFLAGS,RUST_LOG
-    - name: Setup Build Tool
-      if: endsWith(inputs.target, '-linux-musl')
       uses: ./.github/actions/setup_build_tool
       with:
         target: ${{ inputs.target }}

.github/actions/setup_docker/action.yml

@@ -25,7 +25,7 @@ runs:
   using: "composite"
   steps:
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
 
     - name: Configure AWS Credentials
       if: inputs.ecr_role_arn

.github/workflows/reuse.linux.yml

@@ -54,12 +54,12 @@ jobs:
           cat license.json
 
   build:
-    runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
+    runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, "${{ inputs.runner_provider }}"]
     strategy:
       fail-fast: false
       matrix:
         include:
-          - { arch: x86_64, libc: gnu }
+          - { target: x86_64-unknown-linux-gnu, runner: X64 }
     steps:
       - uses: actions/checkout@v4
         with:
@@ -69,16 +69,16 @@ jobs:
         timeout-minutes: 60
         with:
           sha: ${{ github.sha }}
-          target: ${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}
+          target: ${{ matrix.target }}
           artifacts: all
 
   build_address_sanitizer:
-    runs-on: [ self-hosted, ARM64, Linux, 8c16g, "${{ inputs.runner_provider }}" ]
+    runs-on: [ self-hosted, "${{ matrix.runner }}", Linux, 16c32g, "${{ inputs.runner_provider }}" ]
     strategy:
       fail-fast: false
       matrix:
         include:
-          - { arch: aarch64, libc: gnu }
+          - { target: aarch64-unknown-linux-gnu, runner: ARM64 }
     steps:
       - uses: actions/checkout@v4
         with:
@@ -87,27 +87,27 @@ jobs:
       - uses: ./.github/actions/build_linux_sanitizer
         timeout-minutes: 60
         with:
-          target: ${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}
+          target: ${{ matrix.target }}
           artifacts: query
 
-  build_other:
-    runs-on: [self-hosted, X64, Linux, 8c16g, "${{ inputs.runner_provider }}"]
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - { arch: aarch64, libc: musl }
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          # fetch all tags, metasrv and metaclient need tag as its version.
-          fetch-depth: 0
-      - uses: ./.github/actions/build_linux
-        timeout-minutes: 60
-        with:
-          sha: ${{ github.sha }}
-          target: ${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}
-          artifacts: query
+  #build_other:
+  #  runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, "${{ inputs.runner_provider }}"]
+    #strategy:
+      #fail-fast: false
+      #matrix:
+        #include:
+          #- { target: aarch64-unknown-linux-musl, runner: ARM64 }
+    #steps:
+    #  - uses: actions/checkout@v4
+        #with:
+          # #fetch all tags, metasrv and metaclient need tag as its version.
+          #fetch-depth: 0
+     # - uses: ./.github/actions/build_linux
+        #timeout-minutes: 60
+        #with:
+          #sha: ${{ github.sha }}
+          #target: ${{ matrix.target }}
+          #artifacts: query
 
   test_unit:
     runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]

Cargo.toml

@@ -261,4 +261,4 @@ sentry = { git = "https://github.com/getsentry/sentry-rust", rev = "6ef6d97" }
 micromarshal = { git = "https://github.com/ariesdevil/opensrv", rev = "6c96813" }
 async-backtrace = { git = "https://github.com/zhang2014/async-backtrace.git", rev = "dea4553" }
 geozero = { git = "https://github.com/georust/geozero", rev = "1d78b36" }
-#proj = { git = "https://github.com/ariesdevil/proj", rev = "51e1c60" }
+# proj = { git = "https://github.com/ariesdevil/proj", rev = "51e1c60" }

docker/build-tool/alpine/Dockerfile

@@ -23,7 +23,6 @@ RUN curl -sSfLo /tmp/mold.tar.gz https://github.com/rui314/mold/releases/downloa
     rm -rf /tmp/mold.tar.gz && \
     ln -sf /usr/local/bin/mold /usr/bin/$(uname -m)-linux-gnu-ld
 
-ENV LIBCLANG_PATH /usr/lib/llvm17/lib
 ENV RUSTUP_HOME /opt/rust/rustup
 ENV CARGO_HOME /opt/rust/cargo
 ENV PATH /opt/rust/cargo/bin:/opt/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

docker/build-tool/dev/Dockerfile

@@ -1,8 +1,7 @@
 ARG RUST_TOOLCHAIN
 
-FROM datafuselabs/build-tool:base-${RUST_TOOLCHAIN}
+FROM datafuselabs/build-tool:debian-${RUST_TOOLCHAIN}
 
-ARG ARCH
 COPY rust-tools.txt /build/scripts/setup/rust-tools.txt
 RUN /build/scripts/setup/dev_setup.sh -ycd && \
     rm -rf /opt/rust/cargo/git && \

scripts/setup/run_build_tool.sh

@@ -19,21 +19,39 @@ TOOLCHAIN_VERSION=$(awk -F'[ ="]+' '$1 == "channel" { print $2 }' rust-toolchain
 
 _UID=$(id -u)
 _GID=$(id -g)
+USER=${USER:-$(whoami)}
+
+case $TARGET in
+x86_64-unknown-linux-gnu | aarch64-unknown-linux-gnu)
+	RUNNER_BASE="debian"
+	USER_CMD="useradd -u ${_UID} -M -s /bin/bash ${USER}"
+	;;
+x86_64-unknown-linux-musl | aarch64-unknown-linux-musl)
+	RUNNER_BASE="alpine"
+	USER_CMD="adduser -u ${_UID} -D -H -s /bin/bash ${USER}"
+	;;
+*)
+	RUNNER_BASE="dev"
+	TARGET=x86_64-unknown-linux-gnu
+	USER_CMD="useradd -u ${_UID} -M -s /bin/bash ${USER}"
+	;;
+esac
+
 # skip building temporary image for root
 if [[ ${_UID} == 0 ]]; then
-	IMAGE="datafuselabs/build-tool:${TARGET}-${TOOLCHAIN_VERSION}"
+	IMAGE="datafuselabs/build-tool:${RUNNER_BASE}-${TOOLCHAIN_VERSION}"
 else
-	USER=${USER:-$(whoami)}
-	IMAGE="${USER}/build-tool:${TARGET}-${TOOLCHAIN_VERSION}"
+	IMAGE="${USER}/build-tool:${RUNNER_BASE}-${TOOLCHAIN_VERSION}"
 
 	if [[ $(docker image ls "${IMAGE}" --format="true") ]]; then
 		echo "==> build-tool using image ${IMAGE}"
 	else
 		echo "==> preparing temporary build-tool image ${IMAGE} ..."
 		tmpdir=$(mktemp -d)
 		cat >"${tmpdir}/Dockerfile" <<EOF
-FROM datafuselabs/build-tool:${TARGET}-${TOOLCHAIN_VERSION}
-RUN useradd -u ${_UID} ${USER}
+FROM datafuselabs/build-tool:${RUNNER_BASE}-${TOOLCHAIN_VERSION}
+RUN ${USER_CMD}
+RUN chown -R ${USER} /opt/rust/
 RUN printf "${USER} ALL=(ALL:ALL) NOPASSWD:ALL\\n" > /etc/sudoers.d/databend
 EOF
 		docker build -t "${IMAGE}" "${tmpdir}"