05.Version cache - docker dpkg caching support (#12005)

This feature caches all the deb files during docker build and stores them into version cache. It loads the cache file if already exists in the version cache and copies the extracted deb file from cache file into Debian cache path( /var/cache/apt/archives). The apt-install always installs the deb file from the cache if exists, this avoid unnecessary package download from the repo and speeds up the overall build. The cache file is selected based on the SHA value of version dependency files. Why I did it How I did it How to verify it * 03.Version-cache - framework environment settings It defines and passes the necessary version cache environment variables to the caching framework. It adds the utils script for shared cache file access. It also adds the post-cleanup logic for cleaning the unwanted files from the docker/image after the version cache creation. * 04.Version cache - debug framework Added DBGOPT Make variable to enable the cache framework scripts in trace mode. This option takes the part name of the script to enable the particular shell script in trace mode. Multiple shell script names can also be given. Eg: make DBGOPT="image|docker" Added verbose mode to dump the version merge details during build/dry-run mode. Eg: scripts/versions_manager.py freeze -v \ 'dryrun|cmod=docker-swss|cfile=versions-deb|cname=all|stage=sub|stage=add' * 05.Version cache - docker dpkg caching support This feature caches all the deb files during docker build and stores them into version cache. It loads the cache file if already exists in the version cache and copies the extracted deb file from cache file into Debian cache path( /var/cache/apt/archives). The apt-install always installs the deb file from the cache if exists, this avoid unnecessary package download from the repo and speeds up the overall build. The cache file is selected based on the SHA value of version dependency files.
sonic-net · Dec 12, 2022 · 0dc22bd · 0dc22bd
1 parent 5624d15
commit 0dc22bd
Show file tree

Hide file tree

Showing 21 changed files with 586 additions and 76 deletions.
diff --git a/Makefile.work b/Makefile.work
@@ -175,11 +175,22 @@ ifeq ($(ENABLE_FIPS), y)
 endif
 endif
 
+SONIC_VERSION_CACHE := $(filter-out none,$(SONIC_VERSION_CACHE_METHOD))
+SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE)
+SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE)
+export SONIC_VERSION_CACHE SONIC_VERSION_CACHE_SOURCE
+$(shell test -d $(SONIC_VERSION_CACHE_SOURCE) || \
+    mkdir -p $(SONIC_VERSION_CACHE_SOURCE) && chmod -f 777 $(SONIC_VERSION_CACHE_SOURCE) 2>/dev/null )
+
 # Generate the version control build info
 $(shell \
 	SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
 	TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
 	PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
+	DISTRO=$(BLDENV) \
+	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
+	SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \
+	DBGOPT='$(DBGOPT)' \
 	scripts/generate_buildinfo_config.sh)
 
 # Generate the slave Dockerfile, and prepare build info for it
@@ -198,6 +209,8 @@ $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) \
 
 PREPARE_DOCKER=BUILD_SLAVE=y \
 	DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
+	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
+	DBGOPT='$(DBGOPT)' \
 	scripts/prepare_docker_buildinfo.sh \
 		$(SLAVE_BASE_IMAGE) \
 		$(SLAVE_DIR)/Dockerfile \
@@ -220,12 +233,13 @@ SLAVE_TAG = $(shell \
 	(cat $(SLAVE_DIR)/Dockerfile.user \
 		 $(SLAVE_DIR)/Dockerfile \
 		 $(SLAVE_DIR)/buildinfo/versions/versions-* \
-		 .git/HEAD \
 	&& echo $(USER)/$(PWD)/$(CONFIGURED_PLATFORM)) \
 	| sha1sum \
 	| awk '{print substr($$1,0,11);}')
 
 COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
+	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
+	DBGOPT='$(DBGOPT)' \
 	scripts/collect_docker_version_files.sh \
 		$(SLAVE_BASE_IMAGE) \
 		target \
@@ -286,6 +300,10 @@ ifneq ($(SONIC_DPKG_CACHE_SOURCE),)
 	DOCKER_RUN += -v "$(SONIC_DPKG_CACHE_SOURCE):/dpkg_cache:rw"
 endif
 
+ifneq ($(SONIC_VERSION_CACHE_SOURCE),)
+	DOCKER_RUN += -v "$(SONIC_VERSION_CACHE_SOURCE):/vcache:rw"
+endif
+
 ifeq ($(SONIC_ENABLE_SECUREBOOT_SIGNATURE), y)
 ifneq ($(SIGNING_KEY),)
 	DOCKER_SIGNING_SOURCE := $(shell dirname $(SIGNING_KEY))
@@ -379,6 +397,8 @@ DOCKER_SLAVE_BASE_BUILD = docker build --no-cache \
 		    --build-arg http_proxy=$(http_proxy) \
 		    --build-arg https_proxy=$(https_proxy) \
 		    --build-arg no_proxy=$(no_proxy) \
+		    --build-arg SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
+		    --build-arg SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
 		    $(SLAVE_DIR) \
 		    $(SPLIT_LOG) $(DOCKER_BASE_LOG)
 

diff --git a/build_debian.sh b/build_debian.sh
@@ -81,7 +81,10 @@ echo '[INFO] Build host debian base system...'
 TARGET_PATH=$TARGET_PATH scripts/build_debian_base_system.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT
 
 # Prepare buildinfo
-sudo scripts/prepare_debian_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy
+sudo SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE} \
+	DBGOPT="${DBGOPT}" \
+	scripts/prepare_debian_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy
+
 
 sudo chown root:root $FILESYSTEM_ROOT
 
@@ -440,10 +443,10 @@ if [[ $TARGET_BOOTLOADER == grub ]]; then
         GRUB_PKG=grub-efi-arm64-bin
     fi
 
-    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y download \
+    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get install -d -o dir::cache=/var/cache/apt \
         $GRUB_PKG
 
-    sudo mv $FILESYSTEM_ROOT/grub*.deb $FILESYSTEM_ROOT/$PLATFORM_DIR/grub
+    sudo cp $FILESYSTEM_ROOT/var/cache/apt/archives/grub*.deb $FILESYSTEM_ROOT/$PLATFORM_DIR/grub
 fi
 
 ## Disable kexec supported reboot which was installed by default
@@ -635,7 +638,9 @@ if [[ $TARGET_BOOTLOADER == uboot ]]; then
 fi
 
 # Collect host image version files before cleanup
-scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT
+SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE}  \
+	DBGOPT="${DBGOPT}" \
+	scripts/collect_host_image_version_files.sh $CONFIGURED_ARCH $IMAGE_DISTRO $TARGET_PATH $FILESYSTEM_ROOT
 
 # Remove GCC
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y remove gcc

diff --git a/platform/broadcom/sai.mk b/platform/broadcom/sai.mk
@@ -17,4 +17,9 @@ $(BRCM_DNX_SAI)_URL = "$(LIBSAIBCM_DNX_URL_PREFIX)/$(BRCM_DNX_SAI)"
 SONIC_ONLINE_DEBS += $(BRCM_XGS_SAI)
 SONIC_ONLINE_DEBS += $(BRCM_DNX_SAI)
 $(BRCM_XGS_SAI_DEV)_DEPENDS += $(BRCM_XGS_SAI)
+
+$(BRCM_XGS_SAI)_SKIP_VERSION=y
+$(BRCM_XGS_SAI_DEV)_SKIP_VERSION=y
+$(BRCM_DNX_SAI)_SKIP_VERSION=y
+
 $(eval $(call add_conflict_package,$(BRCM_XGS_SAI_DEV),$(LIBSAIVS_DEV)))
diff --git a/rules/config b/rules/config
@@ -229,6 +229,15 @@ TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,htt
 #   docker: docker base images
 SONIC_VERSION_CONTROL_COMPONENTS ?= none
 
+# SONIC_VERSION_CACHE allows the .deb,.py, wget, git, docker and go files to be stored in the cache path. This allows the submodules to
+# cache standard installation package and restored back to avoid the package download every time.
+# SONIC_VERSION_CACHE - Method of deb package caching
+#	none    :  no caching
+#	cache   :  Use cache if exists else build the source and update the cache
+# SONIC_VERSION_CACHE_SOURCE - Defines the version cache location details
+SONIC_VERSION_CACHE_METHOD ?= none
+SONIC_VERSION_CACHE_SOURCE ?= $(SONIC_DPKG_CACHE_SOURCE)/vcache
+
 # SONiC docker registry
 #
 # Set the env variable ENABLE_DOCKER_BASE_PULL = y to enable pulling sonic-slave docker from registry

diff --git a/scripts/build_debian_base_system.sh b/scripts/build_debian_base_system.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
 CONFIGURED_ARCH=$1
 IMAGE_DISTRO=$2
 FILESYSTEM_ROOT=$3

diff --git a/scripts/collect_build_version_files.sh b/scripts/collect_build_version_files.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
 RET=$1
 BLDENV=$2
 TARGET_PATH=$3
@@ -15,12 +17,17 @@ TIMESTAMP=$(date +"%Y%m%d%H%M%S")
 VERSION_BUILD_PATH=$TARGET_PATH/versions/build
 VERSION_SLAVE_PATH=$VERSION_BUILD_PATH/build-sonic-slave-${BLDENV}
 LOG_VERSION_PATH=$VERSION_BUILD_PATH/log-${TIMESTAMP}
+DEFAULT_VERSION_PATH=files/build/versions/default
+BUILD_LOG_PATH=/sonic/target/versions/log/sonic-slave-${BLDENV}/
 
 sudo chmod -R a+rw $BUILDINFO_PATH
 collect_version_files $LOG_VERSION_PATH
 ([ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls $BUILD_VERSION_PATH/)" ]) && cp -rf $BUILD_VERSION_PATH/* $LOG_VERSION_PATH/
 mkdir -p $VERSION_SLAVE_PATH
 
+mkdir -p ${BUILD_LOG_PATH}
+([ -d ${LOG_PATH} ] && [ ! -z "$(ls ${LOG_PATH})" ]) && cp ${LOG_PATH}/* ${BUILD_LOG_PATH}
+
 scripts/versions_manager.py merge -t $VERSION_SLAVE_PATH -b $LOG_VERSION_PATH -e $POST_VERSION_PATH
 
 [ -d $BUILD_VERSION_PATH ] && rm -rf $BUILD_VERSION_PATH/*

diff --git a/scripts/collect_docker_version_files.sh b/scripts/collect_docker_version_files.sh
@@ -1,6 +1,16 @@
 #!/bin/bash
 
-set -x 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
+BUILDINFO_BASE=/usr/local/share/buildinfo
+
+SCRIPT_SRC_PATH=src/sonic-build-hooks
+if [ -e ${SCRIPT_SRC_PATH} ]; then
+	. ${SCRIPT_SRC_PATH}/scripts/utils.sh 
+else
+	. ${BUILDINFO_BASE}/scripts/utils.sh 
+fi
+
 DOCKER_IMAGE=$1
 TARGET_PATH=$2
 DOCKER_IMAGE_TAG=$3
@@ -13,6 +23,8 @@ DOCKER_IMAGE_NAME=$(echo $DOCKER_IMAGE | cut -d: -f1 | sed "s/-$DOCKER_USERNAME\
 #Create the container specific to the user tag and slave tag
 DOCKER_CONTAINER=${DOCKER_IMAGE_TAG/:/-}
 TARGET_VERSIONS_PATH=$TARGET_PATH/versions/dockers/$DOCKER_IMAGE_NAME
+BUILD_LOG_PATH=target/versions/log/$DOCKER_IMAGE_NAME
+mkdir -p ${BUILD_LOG_PATH}
 
 [ -d $TARGET_VERSIONS_PATH ] && rm -rf $TARGET_VERSIONS_PATH
 mkdir -p $TARGET_VERSIONS_PATH
@@ -34,5 +46,57 @@ docker tag ${DOCKER_IMAGE_TAG} tmp-${DOCKER_IMAGE_TAG}
 DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup  --target output -o target/vcache/${DOCKER_IMAGE_NAME} ${DOCKER_PATH}
 DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup  --no-cache --target final --tag ${DOCKER_IMAGE_TAG} ${DOCKER_PATH}
 docker rmi tmp-${DOCKER_IMAGE_TAG}
+docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/log ${BUILD_LOG_PATH}/
+
+
+# Save the cache contents from docker build
+LOCAL_CACHE_FILE=target/vcache/${DOCKER_IMAGE_NAME}/cache.tgz
+CACHE_ENCODE_FILE=${DOCKER_PATH}/vcache/cache.base64
+sleep 1; sync ${CACHE_ENCODE_FILE}
+
+# Decode the cache content into gz format
+SRC_VERSION_PATH=files/build/versions
+if [[ -e ${CACHE_ENCODE_FILE} ]]; then
+
+	cat ${CACHE_ENCODE_FILE} | base64 -d  >${LOCAL_CACHE_FILE}
+	rm -f ${CACHE_ENCODE_FILE}
+fi
+
+# Version package cache
+IMAGE_DBGS_NAME=${DOCKER_IMAGE_NAME//-/_}_image_dbgs
+if [[ ${DOCKER_IMAGE_NAME} == sonic-slave-* ]]; then
+	GLOBAL_CACHE_DIR=${SONIC_VERSION_CACHE_SOURCE}/${DOCKER_IMAGE_NAME}
+else
+	GLOBAL_CACHE_DIR=/vcache/${DOCKER_IMAGE_NAME}
+fi
+
+if [[ ! -z ${SONIC_VERSION_CACHE} && -e ${CACHE_ENCODE_FILE} ]]; then
+
+	# Select version files for SHA calculation
+	VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
+	DEP_FILES="${DOCKER_PATH}/Dockerfile.j2"
+	if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then
+		DEP_FILES="${DEP_FILES} build_debug_docker_j2.sh"
+	fi
+
+	# Calculate the version SHA
+	VERSION_SHA="$( (echo -n "${!IMAGE_DBGS_NAME}"; cat ${DEP_FILES} ${VERSION_FILES}) | sha1sum | awk '{print substr($1,0,23);}')"
+    GLOBAL_CACHE_FILE=${GLOBAL_CACHE_DIR}/${DOCKER_IMAGE_NAME}-${VERSION_SHA}.tgz
+
+	GIT_FILE_STATUS=$(git status -s ${DEP_FILES})
+
+	# If the cache file is not exists in the global cache for the given SHA, 
+	# store the new cache file into version cache path.
+	if [ -f ${LOCAL_CACHE_FILE} ]; then
+		if [[ -z ${GIT_FILE_STATUS} && ! -e ${GLOBAL_CACHE_FILE} ]]; then
+			mkdir -p ${GLOBAL_CACHE_DIR}
+			chmod -f 777 ${GLOBAL_CACHE_DIR}
+			FLOCK ${GLOBAL_CACHE_FILE}
+			cp ${LOCAL_CACHE_FILE} ${GLOBAL_CACHE_FILE} 
+			chmod -f 777 ${LOCAL_CACHE_FILE} ${GLOBAL_CACHE_FILE} 
+			FUNLOCK ${GLOBAL_CACHE_FILE}
+		fi
+	fi
+fi
 
 docker container rm $DOCKER_CONTAINER
diff --git a/scripts/collect_host_image_version_files.sh b/scripts/collect_host_image_version_files.sh
@@ -1,7 +1,16 @@
 #!/bin/bash
 
-TARGET=$1
-FILESYSTEM_ROOT=$2
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
+SCRIPT_SRC_PATH=src/sonic-build-hooks
+if [ -e ${SCRIPT_SRC_PATH} ]; then
+	. ${SCRIPT_SRC_PATH}/scripts/utils.sh 
+fi
+
+ARCH=$1
+DISTRO=$2
+TARGET=$3
+FILESYSTEM_ROOT=$4
 VERSIONS_PATH=$TARGET/versions/host-image
 IMAGENAME="host-image"
 
@@ -13,3 +22,5 @@ sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo ${IMAGENAME}
 
 cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/pre-versions $VERSIONS_PATH/
 cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/post-versions $VERSIONS_PATH/
+
+sudo LANG=C chroot $FILESYSTEM_ROOT post_run_cleanup ${IMAGENAME}
diff --git a/scripts/generate_buildinfo_config.sh b/scripts/generate_buildinfo_config.sh
@@ -6,5 +6,8 @@ BUILDINFO_CONFIG=$BUILDINFO_PATH/buildinfo/config/buildinfo.config
 
 mkdir -p $BUILDINFO_PATH/buildinfo/config
 
-echo "PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG
-echo "SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG
+echo "export PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG
+echo "export SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG
+echo "export SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE}" >> $BUILDINFO_CONFIG
+echo "export SONIC_VERSION_CACHE_SOURCE=${SONIC_VERSION_CACHE_SOURCE}" >> $BUILDINFO_CONFIG
+echo "export DISTRO=${DISTRO}" >> $BUILDINFO_CONFIG
diff --git a/scripts/prepare_debian_image_buildinfo.sh b/scripts/prepare_debian_image_buildinfo.sh
@@ -1,9 +1,13 @@
 #!/bin/bash
 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
 
 ARCH=$1
 DISTRO=$2
 FILESYSTEM_ROOT=$3
+HOST_IMAGE_NAME=host-image
+IMAGENAME=${HOST_IMAGE_NAME}
 
 . /usr/local/share/buildinfo/scripts/buildinfo_base.sh
 VERSION_DEB_PREFERENCE="01-versions-deb"
@@ -26,4 +30,8 @@ if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then
 fi
 
 sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb"
-sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo"
+#sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo"
+sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo export DISTRO=${DISTRO} >> /usr/local/share/buildinfo/config/buildinfo.config"
+sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo export IMAGENAME=${IMAGENAME} >> /usr/local/share/buildinfo/config/buildinfo.config"
+
+sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo ${HOST_IMAGE_NAME}"