Add amazonlinux image, prepare to remove Centos7 image, remove openblas image (milvus-io#26368)

Signed-off-by: Enwei Jiao <[email protected]>

Author: jiaoew1991

.github/mergify.yml

@@ -29,17 +29,11 @@ on:
 
 jobs:
   ubuntu:
-    name: ${{ matrix.name }}
-    runs-on: ${{ matrix.os }}
+    name: Code Checker AMD64 Ubuntu 20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 180
     strategy:
       fail-fast: false
-      matrix:
-        include:
-          - name: Code Checker AMD64 Ubuntu 20.04
-            os: ubuntu-20.04
-    env:
-      UBUNTU: 20.04
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -50,7 +44,6 @@ jobs:
           echo "corehash=${CORE_HASH}" >> $GITHUB_ENV
           echo "Set CCache hash to ${CORE_HASH}"
       - name: Cache CCache Volumes
-        # uses: actions/cache@v3
         uses: pat-s/always-upload-cache@v3
         with:
           path: .docker/amd64-ubuntu20.04-ccache
@@ -63,21 +56,22 @@ jobs:
           key: ubuntu20.04-go-mod-${{ hashFiles('**/go.sum') }}
           restore-keys: ubuntu20.04-go-mod-
       - name: Cache Conan Packages
-        # uses: actions/cache@v3
         uses: pat-s/always-upload-cache@v3
         with:
           path: .docker/amd64-ubuntu20.04-conan
           key: ubuntu20.04-conan-${{ hashFiles('internal/core/conanfile.*') }}
           restore-keys: ubuntu20.04-conan-
+      # - name: Setup upterm session
+      #   uses: lhotari/action-upterm@v1
       - name: Code Check
         env:
           OS_NAME: 'ubuntu20.04'
         run: |
           ./build/builder.sh /bin/bash -c "make check-proto-product && make verifiers"
-  centos:
-    name: Code Checker CentOS 7
-    # Run in centos docker
-    runs-on: ubuntu-20.04
+  amazonlinux:
+    name: Code Checker Amazonlinux 2023
+    # Run in amazonlinux docker
+    runs-on: ubuntu-latest
     timeout-minutes: 180
     steps:
       - name: Checkout
@@ -89,33 +83,31 @@ jobs:
           echo "corehash=${CORE_HASH}" >> $GITHUB_ENV
           echo "Set CCache hash to ${CORE_HASH}"
       - name: Cache CCache Volumes
-        # uses: actions/cache@v3
         uses: pat-s/always-upload-cache@v3
         with:
-          path: .docker/amd64-centos7-ccache
-          key: centos7-ccache-${{ env.corehash }}
-          restore-keys: centos7-ccache-
+          path: .docker/amd64-amazonlinux2023-ccache
+          key: amazonlinux2023-ccache-${{ env.corehash }}
+          restore-keys: amazonlinux2023-ccache-
       - name: Cache Third Party
         uses: actions/cache@v3
         with:
           path: .docker/thirdparty
-          key: centos7-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }}
-          restore-keys: centos7-thirdparty-
+          key: amazonlinux2023-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }}
+          restore-keys: amazonlinux2023-thirdparty-
       - name: Cache Go Mod Volumes
         uses: actions/cache@v3
         with:
-          path: .docker/amd64-centos7-go-mod
-          key: centos7-go-mod-${{ hashFiles('**/go.sum') }}
-          restore-keys: centos7-go-mod-
+          path: .docker/amd64-amazonlinux2023-go-mod
+          key: amazonlinux2023-go-mod-${{ hashFiles('**/go.sum') }}
+          restore-keys: amazonlinux2023-go-mod-
       - name: Cache Conan Packages
-        # uses: actions/cache@v3
         uses: pat-s/always-upload-cache@v3
         with:
-          path: .docker/amd64-centos7-conan
-          key: centos7-conan-${{ hashFiles('internal/core/conanfile.*') }}
-          # restore-keys: centos7-conan-
+          path: .docker/amd64-amazonlinux2023-conan
+          key: amazonlinux2023-conan-${{ hashFiles('internal/core/conanfile.*') }}
+          restore-keys: amazonlinux2023-conan-
       - name: Code Check
         env:
-          OS_NAME: 'centos7'
+          OS_NAME: 'amazonlinux2023'
         run: |
           ./build/builder.sh /bin/bash -c "make install"

.github/workflows/code-checker.yaml

@@ -18,15 +18,17 @@ on:
 
 jobs:
   publish-builder:
-    name: AMD64 ${{ matrix.os }}
+    name: ${{ matrix.arch }} ${{ matrix.os }}
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu20.04, centos7]
+        os: [ubuntu20.04, amazonlinux2023]
+        arch: [amd64]
     env:
       OS_NAME: ${{ matrix.os }}
+      IMAGE_ARCH: ${{ matrix.arch }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -35,6 +37,8 @@ jobs:
         run: |
           echo "::set-output name=version::$(date +%Y%m%d)"
           echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      # - name: Setup upterm session
+      #   uses: lhotari/action-upterm@v1
       - name: Docker Pull
         shell: bash
         run: |

.github/workflows/publish-builder.yaml

@@ -28,20 +28,21 @@ OS_NAME="${OS_NAME:-ubuntu20.04}"
 MILVUS_IMAGE_REPO="${MILVUS_IMAGE_REPO:-milvusdb/milvus}"
 MILVUS_IMAGE_TAG="${MILVUS_IMAGE_TAG:-latest}"
 
-MILVUS_BASE_IMAGE_REPO="${MILVUS_BASE_IMAGE_REPO:-milvusdb/milvus-base}"
-MILVUS_BASE_IMAGE_TAG="local"
+if [ -z "$IMAGE_ARCH" ]; then
+    MACHINE=$(uname -m)
+    if [ "$MACHINE" = "x86_64" ]; then
+        IMAGE_ARCH="amd64"
+    else
+        IMAGE_ARCH="arm64"
+    fi
+
+fi
+
 BUILD_ARGS=""
 
 pushd "${toplevel}"
-BUILD_BASE_IMAGE=${BUILD_BASE_IMAGE:-"false"}
-
-# Seperate base dockerfile to ignore install dependencies when build milvus image
-if [[ ${OS_NAME} == "ubuntu20.04" && ${BUILD_BASE_IMAGE} == "true" ]]; then 
-  docker build -f "./build/docker/milvus/${OS_NAME}/Dockerfile.base" -t "${MILVUS_BASE_IMAGE_REPO}:${MILVUS_BASE_IMAGE_TAG}" .
-  BUILD_ARGS="--build-arg MILVUS_BASE_IMAGE_REPO=${MILVUS_BASE_IMAGE_REPO} --build-arg MILVUS_BASE_IMAGE_TAG=${MILVUS_BASE_IMAGE_TAG}"
-fi 
 
-docker build ${BUILD_ARGS} -f "./build/docker/milvus/${OS_NAME}/Dockerfile" -t "${MILVUS_IMAGE_REPO}:${MILVUS_IMAGE_TAG}" .
+docker build ${BUILD_ARGS} --platform linux/${IMAGE_ARCH} -f "./build/docker/milvus/${OS_NAME}/Dockerfile" -t "${MILVUS_IMAGE_REPO}:${MILVUS_IMAGE_TAG}" .
 
 image_size=$(docker inspect ${MILVUS_IMAGE_REPO}:${MILVUS_IMAGE_TAG}  -f '{{.Size}}'| awk '{ byte =$1 /1024/1024/1024; print byte " GB" }')
 

.github/workflows/publish-milvus-base.yaml

@@ -5,6 +5,10 @@ set -euo pipefail
 # Absolute path to the toplevel milvus directory.
 toplevel=$(dirname "$(cd "$(dirname "${0}")"; pwd)")
 
+if [[ -f "$toplevel/.env" ]]; then
+  export $(cat $toplevel/.env | xargs)
+fi
+
 export OS_NAME="${OS_NAME:-ubuntu20.04}"
 
 pushd "${toplevel}"
@@ -19,34 +23,17 @@ if [[ "${1-}" == "down" ]]; then
     exit 0
 fi
 
-# Attempt to run in the container with the same UID/GID as we have on the host,
-# as this results in the correct permissions on files created in the shared
-# volumes. This isn't always possible, however, as IDs less than 100 are
-# reserved by Debian, and IDs in the low 100s are dynamically assigned to
-# various system users and groups. To be safe, if we see a UID/GID less than
-# 500, promote it to 501. This is notably necessary on macOS Lion and later,
-# where administrator accounts are created with a GID of 20. This solution is
-# not foolproof, but it works well in practice.
-uid=$(id -u)
-gid=$(id -g)
-[ "$uid" -lt 500 ] && uid=501
-[ "$gid" -lt 500 ] && gid=$uid
-
-mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-${OS_NAME}-ccache"
-mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-${OS_NAME}-go-mod"
-mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-${OS_NAME}-vscode-extensions"
-mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-${OS_NAME}-conan"
+mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-ccache"
+mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-go-mod"
+mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-vscode-extensions"
+mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-conan"
 chmod -R 777 "${DOCKER_VOLUME_DIRECTORY:-.docker}"
 
 docker-compose pull --ignore-pull-failures builder
-if [[ "${CHECK_BUILDER:-}" == "1" ]]; then
+if [[ "${CHECK_BUILDER:-1}" == "1" ]]; then
     docker-compose build builder
 fi
 
-if [[ "$(id -u)" != "0" ]]; then
-    docker-compose run --no-deps --rm -u "$uid:$gid" builder "$@"
-else
-    docker-compose run --no-deps --rm --entrypoint "/tini -- /entrypoint.sh" builder "$@"
-fi
+docker-compose run --no-deps --rm builder "$@"
 
 popd

build/build_image.sh

@@ -0,0 +1,50 @@
+# Copyright (C) 2019-2022 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+
+FROM amazonlinux:2023
+
+ARG TARGETARCH
+
+RUN yum install -y wget g++ gcc gdb libstdc++-static git make unzip tar which \
+    autoconf automake golang python3 python3-pip perl-FindBin texinfo \
+    pkg-config libuuid-devel libaio && \
+    rm -rf /var/cache/yum/*
+
+RUN pip3 install conan==1.58.0
+
+RUN echo "target arch $TARGETARCH"
+RUN if [ "$TARGETARCH" = "amd64" ]; then CMAKE_SUFFIX=x86_64; else CMAKE_SUFFIX=aarch64; fi &&\
+    wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.4-linux-$CMAKE_SUFFIX.tar.gz" | tar --strip-components=1 -xz -C /usr/local
+
+RUN mkdir /tmp/ccache && wget -qO- https://github.com/ccache/ccache/releases/download/v4.8.2/ccache-4.8.2.tar.gz | tar --strip-components=1 -xz -C /tmp/ccache &&\
+    cd /tmp/ccache && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr .. && make && make install &&\
+    rm -rf /tmp/ccache
+
+# https://github.com/golang/go/issues/22040 Due to this issue, ld.gold cannot be used in the ARM environment, so the official golang package cannot be used. You need to use the golang package that comes with yum.
+# Install Go
+# ENV GOPATH /go
+# ENV GOROOT /usr/local/go
+# ENV GO111MODULE on
+# ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+# RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.20.7.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
+#     mkdir -p "$GOPATH/src" "$GOPATH/bin" && \
+#     go clean --modcache && \
+#     chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR)
+
+# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild
+RUN mkdir -p /home/milvus/.vscode-server/extensions \
+        /home/milvus/.vscode-server-insiders/extensions \
+    && chmod -R 777 /home/milvus
+
+COPY --chown=0:0 build/docker/builder/entrypoint.sh /
+
+ENTRYPOINT [ "/entrypoint.sh" ]
+CMD ["tail", "-f", "/dev/null"]