docker: use multi-stage build

.azure-pipelines/pipelines.yml

@@ -422,8 +422,7 @@ stages:
       inputs:
         buildType: current
         artifactName: "docker"
-        itemPattern: "docker/envoy-docker-images.tar.xz"
-        downloadType: single
+        itemPattern: "docker/envoy*.tar"
         targetPath: $(Build.StagingDirectory)
     - bash: ./ci/do_ci.sh verify_examples
       env:
@@ -438,8 +437,7 @@ stages:
       inputs:
         buildType: current
         artifactName: "docker"
-        itemPattern: "docker/envoy-docker-images.tar.xz"
-        downloadType: single
+        itemPattern: "docker/envoy*.tar"
         targetPath: $(Build.StagingDirectory)
     - bash: ./ci/do_ci.sh verify_build_examples
       env:

ci/Dockerfile-envoy

@@ -1,9 +1,21 @@
 ARG BUILD_OS=ubuntu
 ARG BUILD_TAG=18.04
+ARG ENVOY_VRP_BASE_IMAGE=envoy
+
+
+FROM scratch AS binary
 
-# Final stage
-FROM $BUILD_OS:$BUILD_TAG
 ARG TARGETPLATFORM
+ARG ENVOY_BINARY=envoy
+ARG ENVOY_BINARY_SUFFIX=_stripped
+ADD ${TARGETPLATFORM}/build_${ENVOY_BINARY}_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/
+ADD configs/envoyproxy_io_proxy.yaml /etc/envoy/envoy.yaml
+COPY --chown=root:root ${TARGETPLATFORM}/build_${ENVOY_BINARY}_release/su-exec /usr/local/bin/
+COPY ci/docker-entrypoint.sh /
+
+
+# STAGE: envoy
+FROM ${BUILD_OS}:${BUILD_TAG} AS envoy
 
 RUN apt-get update && apt-get upgrade -y \
     && apt-get install --no-install-recommends -y ca-certificates \
@@ -13,16 +25,76 @@ RUN apt-get update && apt-get upgrade -y \
 
 RUN mkdir -p /etc/envoy
 
-ARG ENVOY_BINARY=envoy
-ARG ENVOY_BINARY_SUFFIX=_stripped
-ADD ${TARGETPLATFORM}/build_${ENVOY_BINARY}_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/
-ADD configs/envoyproxy_io_proxy.yaml /etc/envoy/envoy.yaml
+COPY --from=binary /usr/local/bin/* /usr/local/bin/
+COPY --from=binary /etc/envoy/envoy.yaml /etc/envoy/envoy.yaml
+COPY --from=binary /docker-entrypoint.sh /
+
+RUN adduser --group --system envoy
+
+EXPOSE 10000
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["envoy", "-c", "/etc/envoy/envoy.yaml"]
+
+
+# STAGE: envoy-distroless
+FROM gcr.io/distroless/base-debian11:nonroot AS envoy-distroless
 
-ADD ${TARGETPLATFORM}/build_${ENVOY_BINARY}_release/su-exec /usr/local/bin/
-RUN chown root:root /usr/local/bin/su-exec && adduser --group --system envoy
+COPY --from=binary /usr/local/bin/* /usr/local/bin/
+COPY --from=binary /etc/envoy/envoy.yaml /etc/envoy/envoy.yaml
+
+EXPOSE 10000
+
+ENTRYPOINT ["/usr/local/bin/envoy"]
+CMD ["-c", "/etc/envoy/envoy.yaml"]
+
+
+# STAGE: envoy-alpine
+# TODO(https://github.com/envoyproxy/envoy/issues/19781): Deprecate and remove this image
+FROM frolvlad/alpine-glibc:alpine-3.14_glibc-2.33 AS envoy-alpine
+
+RUN mkdir -p /etc/envoy
+
+COPY --from=binary /usr/local/bin/envoy /usr/local/bin/
+COPY --from=binary /etc/envoy/envoy.yaml /etc/envoy/envoy.yaml
+
+RUN apk add --no-cache shadow su-exec \
+    && addgroup -S envoy && adduser --no-create-home -S envoy -G envoy
 
 EXPOSE 10000
 
 COPY ci/docker-entrypoint.sh /
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["envoy", "-c", "/etc/envoy/envoy.yaml"]
+
+
+# STAGE: envoy-google-vrp
+FROM ${ENVOY_VRP_BASE_IMAGE} AS envoy-google-vrp
+
+RUN apt-get update \
+    && apt-get upgrade -y \
+    && apt-get install -y libc++1 supervisor gdb strace tshark \
+    && apt-get autoremove -y \
+    && apt-get clean \
+    && rm -rf /tmp/* /var/tmp/* \
+    && rm -rf /var/lib/apt/lists/*
+
+ADD configs/google-vrp/envoy-edge.yaml /etc/envoy/envoy-edge.yaml
+ADD configs/google-vrp/envoy-origin.yaml /etc/envoy/envoy-origin.yaml
+ADD configs/google-vrp/launch_envoy.sh /usr/local/bin/launch_envoy.sh
+ADD configs/google-vrp/supervisor.conf /etc/supervisor.conf
+ADD test/config/integration/certs/serverkey.pem /etc/envoy/certs/serverkey.pem
+ADD test/config/integration/certs/servercert.pem /etc/envoy/certs/servercert.pem
+# ADD %local envoy bin% /usr/local/bin/envoy
+RUN chmod 777 /var/log/supervisor
+RUN chmod a+r /etc/supervisor.conf /etc/envoy/* /etc/envoy/certs/*
+RUN chmod a+rx /usr/local/bin/launch_envoy.sh
+
+EXPOSE 10000
+EXPOSE 10001
+
+CMD ["supervisord", "-c", "/etc/supervisor.conf"]
+
+
+# Make envoy image as last stage so it is built by default
+FROM envoy

ci/do_ci.sh

@@ -158,18 +158,19 @@ function run_process_test_result() {
 
 function run_ci_verify () {
   echo "verify examples..."
-  docker load < "$ENVOY_DOCKER_BUILD_DIR/docker/envoy-docker-images.tar.xz"
-  _images=$(docker image list --format "{{.Repository}}")
-  while read -r line; do images+=("$line"); done \
-      <<< "$_images"
-  _tags=$(docker image list --format "{{.Tag}}")
-  while read -r line; do tags+=("$line"); done \
-      <<< "$_tags"
-  for i in "${!images[@]}"; do
-      if [[ "${images[i]}" =~ "envoy" ]]; then
-          docker tag "${images[$i]}:${tags[$i]}" "${images[$i]}:latest"
-      fi
+  OCI_TEMP_DIR="${ENVOY_DOCKER_BUILD_DIR}/image"
+  mkdir -p "${OCI_TEMP_DIR}"
+
+  IMAGES=("envoy" "envoy-contrib" "envoy-google-vrp")
+
+  for IMAGE in "${IMAGES[@]}"; do
+    tar xvf "${ENVOY_DOCKER_BUILD_DIR}/docker/${IMAGE}.tar" -C "${OCI_TEMP_DIR}"
+    skopeo copy "oci:${OCI_TEMP_DIR}" "docker-daemon:envoyproxy/${IMAGE}-dev:latest"
+    rm -rf "${OCI_TEMP_DIR:?}/*"
   done
+
+  rm -rf "${OCI_TEMP_DIR:?}"
+
   docker images
   sudo apt-get update -y
   sudo apt-get install -y -qq --no-install-recommends expect redis-tools

ci/docker_ci.sh

@@ -36,13 +36,13 @@ build_platforms() {
 
 build_args() {
   TYPE=$1
-  FILE_SUFFIX="${TYPE/-debug/}"
-  FILE_SUFFIX="${FILE_SUFFIX/-contrib/}"
-  FILE_SUFFIX="${FILE_SUFFIX/-ltsc2022/}"
 
-  printf ' -f ci/Dockerfile-envoy%s' "${FILE_SUFFIX}"
   if [[ "${TYPE}" == *-windows* ]]; then
-   printf ' --build-arg BUILD_OS=%s --build-arg BUILD_TAG=%s' "${WINDOWS_IMAGE_BASE}" "${WINDOWS_IMAGE_TAG}"
+    printf ' -f ci/Dockerfile-envoy-windows --build-arg BUILD_OS=%s --build-arg BUILD_TAG=%s' "${WINDOWS_IMAGE_BASE}" "${WINDOWS_IMAGE_TAG}"
+  else
+    TARGET="${TYPE/-debug/}"
+    TARGET="${TARGET/-contrib/}"
+    printf ' -f ci/Dockerfile-envoy --target %s' "envoy${TARGET}"
   fi
 
   if [[ "${TYPE}" == *-contrib* ]]; then
@@ -51,25 +51,17 @@ build_args() {
 
   if [[ "${TYPE}" == *-debug ]]; then
       printf ' --build-arg ENVOY_BINARY_SUFFIX='
-  elif [[ "${TYPE}" == "-google-vrp" ]]; then
-      printf ' --build-arg ENVOY_VRP_BASE_IMAGE=%s' "${VRP_BASE_IMAGE}"
   fi
 }
 
 use_builder() {
   # BuildKit is not available for Windows images, skip this
   if ! is_windows; then
     TYPE=$1
-    if [[ "${TYPE}" == "-google-vrp" ]]; then
-      docker buildx use default
-    else
-      docker buildx use multi-builder
-    fi
+    docker buildx use multi-builder
   fi
 }
 
-IMAGES_TO_SAVE=()
-
 build_images() {
   local _args args=()
   TYPE=$1
@@ -80,21 +72,11 @@ build_images() {
   read -ra args <<< "$_args"
   PLATFORM="$(build_platforms "${TYPE}")"
 
-  docker "${BUILD_COMMAND[@]}" --platform "${PLATFORM}" "${args[@]}" -t "${BUILD_TAG}" .
+  if ! is_windows && ! [[ "${TYPE}" =~ debug ]]; then
+    args+=("-o" "type=oci,dest=${ENVOY_DOCKER_IMAGE_DIRECTORY}/envoy${TYPE}.tar")
+  fi
 
-  PLATFORM="$(build_platforms "${TYPE}" | tr ',' ' ')"
-  for ARCH in ${PLATFORM}; do
-    if [[ "${ARCH}" == "linux/amd64" ]] || [[ "${ARCH}" == "windows/amd64" ]]; then
-      IMAGE_TAG="${BUILD_TAG}"
-    else
-      IMAGE_TAG="${BUILD_TAG}-${ARCH/linux\//}"
-    fi
-
-    # docker buildx load cannot have multiple platform, load individually
-    if ! is_windows; then
-      docker "${BUILD_COMMAND[@]}" --platform "${ARCH}" "${args[@]}" -t "${IMAGE_TAG}" . --load
-    fi
-  done
+  docker "${BUILD_COMMAND[@]}" --platform "${PLATFORM}" "${args[@]}" -t "${BUILD_TAG}" .
 }
 
 push_images() {
@@ -127,7 +109,7 @@ fi
 
 # This prefix is altered for the private security images on setec builds.
 DOCKER_IMAGE_PREFIX="${DOCKER_IMAGE_PREFIX:-envoyproxy/envoy}"
-
+mkdir -p "${ENVOY_DOCKER_IMAGE_DIRECTORY}"
 
 if is_windows; then
   BUILD_TYPES=("-${WINDOWS_BUILD_TYPE}")
@@ -140,34 +122,15 @@ else
   # Configure docker-buildx tools
   BUILD_COMMAND=("buildx" "build")
   config_env
-
-  # VRP base image is only for Linux amd64
-  VRP_BASE_IMAGE="${DOCKER_IMAGE_PREFIX}${IMAGE_POSTFIX}:${IMAGE_NAME}"
 fi
 
 # Test the docker build in all cases, but use a local tag that we will overwrite before push in the
 # cases where we do push.
 for BUILD_TYPE in "${BUILD_TYPES[@]}"; do
     image_tag="${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:${IMAGE_NAME}"
     build_images "${BUILD_TYPE}" "$image_tag"
-
-    if ! is_windows; then
-        if [[ "$BUILD_TYPE" == "" || "$BUILD_TYPE" == "-contrib" || "$BUILD_TYPE" == "-alpine" || "$BUILD_TYPE" == "-distroless" ]]; then
-            # verify_examples expects the base and alpine images, and for them to be named `-dev`
-            dev_image="envoyproxy/envoy${BUILD_TYPE}-dev:latest"
-            docker tag "$image_tag" "$dev_image"
-            IMAGES_TO_SAVE+=("$dev_image")
-        fi
-    fi
 done
 
-mkdir -p "${ENVOY_DOCKER_IMAGE_DIRECTORY}"
-if [[ ${#IMAGES_TO_SAVE[@]} -ne 0 ]]; then
-    ENVOY_DOCKER_TAR="${ENVOY_DOCKER_IMAGE_DIRECTORY}/envoy-docker-images.tar.xz"
-    echo "Saving built images to ${ENVOY_DOCKER_TAR}: ${IMAGES_TO_SAVE[*]}"
-    docker save "${IMAGES_TO_SAVE[@]}" | xz -T0 -2 >"${ENVOY_DOCKER_TAR}"
-fi
-
 # Only push images for main builds, release branch builds, and tag builds.
 if [[ "${AZP_BRANCH}" != "${MAIN_BRANCH}" ]] &&
   ! [[ "${AZP_BRANCH}" =~ ${RELEASE_BRANCH_REGEX} ]] &&

ci/docker_rebuild_google-vrp.sh

@@ -25,8 +25,8 @@ BASE_DOCKER_IMAGE="envoyproxy/envoy-dev:latest"
 
 BUILD_DIR="$(mktemp -d)"
 declare -r BUILD_DIR
-cp ci/Dockerfile-envoy-google-vrp "${BUILD_DIR}"
-declare -r DOCKER_BUILD_FILE="${BUILD_DIR}"/Dockerfile-envoy-google-vrp
+cp ci/Dockerfile-envoy "${BUILD_DIR}"
+declare -r DOCKER_BUILD_FILE="${BUILD_DIR}"/Dockerfile-envoy
 
 # If we have a local Envoy binary, use a variant of the build environment that supports it.
 if [[ -n "$1" ]]; then
@@ -43,7 +43,7 @@ fi
 
 cat "${DOCKER_BUILD_FILE}"
 
-docker build -t "envoy-google-vrp:local" --build-arg "ENVOY_VRP_BASE_IMAGE=${BASE_DOCKER_IMAGE}" -f "${DOCKER_BUILD_FILE}" .
+DOCKER_BUILDKIT=1 docker build -t "envoy-google-vrp:local" --target envoy-google-vrp --build-arg "ENVOY_VRP_BASE_IMAGE=${BASE_DOCKER_IMAGE}" -f "${DOCKER_BUILD_FILE}" .
 
 if [[ -n "$1" ]]; then
   rm -rf "${LOCAL_ENVOY_DIR}"