From 2a205599121557ce6ebdb0857222a8aeec452a49 Mon Sep 17 00:00:00 2001 From: xumia <59720581+xumia@users.noreply.github.com> Date: Fri, 13 Jan 2023 16:16:35 +0800 Subject: [PATCH] [Build] Support Debian snapshot mirror to improve build stability (#13097) Why I did it [Build] Support Debian snapshot mirror to improve build stability It is to enhance the reproducible build, supports the Debian snapshot mirror. It guarantees all the docker images using the same Debian mirror snapshot and fixes the temporary build failure which is caused by remote Debain mirror indexes changed during the build. It is also to fix the version conflict issue caused by no fixed versions of some of the Debian packages. How I did it Add a new feature to support the Debian snapshot mirror. How to verify it --- rules/config | 3 ++ scripts/build_mirror_config.sh | 28 +++++++++++++++++++ scripts/generate_buildinfo_config.sh | 1 + scripts/versions_manager.py | 9 ++++-- slave.mk | 1 + .../scripts/buildinfo_base.sh | 27 +++++++++++++++++- .../scripts/collect_version_files | 10 +++++++ 7 files changed, 75 insertions(+), 4 deletions(-) diff --git a/rules/config b/rules/config index cc67914a807e..5daa2f933a57 100644 --- a/rules/config +++ b/rules/config @@ -186,6 +186,9 @@ TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,htt # docker: docker base images SONIC_VERSION_CONTROL_COMPONENTS ?= none +# MIRROR_SNAPSHOT - support mirror snapshot flag +MIRROR_SNAPSHOT ?= n + # SONiC docker registry # # Set the env variable ENABLE_DOCKER_BASE_PULL = y to enable pulling sonic-slave docker from registry diff --git a/scripts/build_mirror_config.sh b/scripts/build_mirror_config.sh index 5f94e701cbde..a1e5900a48b8 100755 --- a/scripts/build_mirror_config.sh +++ b/scripts/build_mirror_config.sh @@ -5,16 +5,40 @@ CONFIG_PATH=$1 export ARCHITECTURE=$2 export DISTRIBUTION=$3 +DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net +MIRROR_VERSION_FILE=files/build/versions/default/versions-mirror +[ -f target/versions/default/versions-mirror ] && MIRROR_VERSION_FILE=target/versions/default/versions-mirror + # The default mirror urls DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/ DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/ + # The debian-archive.trafficmanager.net does not support armhf, use debian.org instead if [ "$ARCHITECTURE" == "armhf" ]; then DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/ DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/ fi +if [ "$MIRROR_SNAPSHOT" == y ]; then + if [ -f $MIRROR_VERSION_FILE ]; then + DEBIAN_TIMESTAMP=$(grep "^debian==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//') + DEBIAN_SECURITY_TIMESTAMP=$(grep "^debian-security==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//') + elif [ -z "$DEBIAN_TIMESTAMP" ] || [ -z "$DEBIAN_SECURITY_TIMESTAMP" ]; then + DEBIAN_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian/latest/timestamp) + DEBIAN_SECURITY_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian-security/latest/timestamp) + fi + + DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/snapshot/debian/$DEBIAN_TIMESTAMP/ + DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/snapshot/debian-security/$DEBIAN_SECURITY_TIMESTAMP/ + + mkdir -p target/versions/default + if [ ! -f target/versions/default/versions-mirror ]; then + echo "debian==$DEBIAN_TIMESTAMP" > target/versions/default/versions-mirror + echo "debian-security==$DEBIAN_SECURITY_TIMESTAMP" >> target/versions/default/versions-mirror + fi +fi + [ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS [ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS @@ -24,3 +48,7 @@ TEMPLATE=files/apt/sources.list.j2 [ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2 MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE +if [ "$MIRROR_SNAPSHOT" == y ]; then + # Set the snapshot mirror, and add the SET_REPR_MIRRORS flag + sed -i -e "/^#*deb.*packages.trafficmanager.net/! s/^#*deb/#&/" -e "\$a#SET_REPR_MIRRORS" $CONFIG_PATH/sources.list.$ARCHITECTURE +fi diff --git a/scripts/generate_buildinfo_config.sh b/scripts/generate_buildinfo_config.sh index fe7657a6b6c9..080e6a321a6b 100755 --- a/scripts/generate_buildinfo_config.sh +++ b/scripts/generate_buildinfo_config.sh @@ -8,3 +8,4 @@ mkdir -p $BUILDINFO_PATH/buildinfo/config echo "PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG echo "SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG +echo "export MIRROR_SNAPSHOT=$MIRROR_SNAPSHOT" >> $BUILDINFO_CONFIG diff --git a/scripts/versions_manager.py b/scripts/versions_manager.py index a20684e97bb4..d4127dece932 100755 --- a/scripts/versions_manager.py +++ b/scripts/versions_manager.py @@ -475,12 +475,13 @@ def filter(self, ctypes=[]): module.filter(ctypes=ctypes) def get_default_module(self): - if DEFAULT_MODULE in self.modules: - return self.modules[DEFAULT_MODULE] + default_module = self.modules.get(DEFAULT_MODULE, VersionModule(DEFAULT_MODULE, [])) ctypes = self.get_component_types() dists = self.get_dists() components = [] for ctype in ctypes: + if ctype in DEFAULT_OVERWRITE_COMPONENTS: + continue if ctype == 'deb': for dist in dists: versions = self._get_versions(ctype, dist) @@ -492,7 +493,9 @@ def get_default_module(self): common_versions = self._get_common_versions(versions) component = Component(common_versions, ctype) components.append(component) - return VersionModule(DEFAULT_MODULE, components) + module = VersionModule(DEFAULT_MODULE, components) + module.overwrite(default_module, True, True) + return module def get_aggregatable_modules(self): modules = {} diff --git a/slave.mk b/slave.mk index 89def3169295..0bd9912d8f82 100644 --- a/slave.mk +++ b/slave.mk @@ -75,6 +75,7 @@ export IMAGE_DISTRO export IMAGE_DISTRO_DEBS_PATH export MULTIARCH_QEMU_ENVIRON export DOCKER_BASE_ARCH +export MIRROR_SNAPSHOT ############################################################################### ## Utility rules diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh index 4c94eaf0aa4e..ba8790128ecc 100755 --- a/src/sonic-build-hooks/scripts/buildinfo_base.sh +++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh @@ -11,7 +11,7 @@ POST_VERSION_PATH=$BUILDINFO_PATH/post-versions VERSION_DEB_PREFERENCE=$BUILDINFO_PATH/versions/01-versions-deb WEB_VERSION_FILE=$VERSION_PATH/versions-web BUILD_WEB_VERSION_FILE=$BUILD_VERSION_PATH/versions-web -REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/debian' +REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/' DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock . $BUILDINFO_PATH/config/buildinfo.config @@ -72,14 +72,39 @@ set_reproducible_mirrors() { # Remove the charater # in front of the line if matched local expression="s/^#\(.*$REPR_MIRROR_URL_PATTERN\)/\1/" + # Add the character # in front of the line, if not match the URL pattern condition + local expression2="/^#*deb.*$REPR_MIRROR_URL_PATTERN/! s/^#*deb/#&/" + local expression3="\$a#SET_REPR_MIRRORS" if [ "$1" = "-d" ]; then # Add the charater # in front of the line if match expression="s/^deb.*$REPR_MIRROR_URL_PATTERN/#\0/" + # Remove the character # in front of the line, if not match the URL pattern condition + expression2="/^#*deb.*$REPR_MIRROR_URL_PATTERN/! s/^#(#*deb)/\1/" + expression3="/#SET_REPR_MIRRORS/d" fi local mirrors="/etc/apt/sources.list $(find /etc/apt/sources.list.d/ -type f)" for mirror in $mirrors; do + if ! grep -iq "$REPR_MIRROR_URL_PATTERN" "$mirror"; then + continue + fi + + # Make sure no duplicate operations on the mirror config file + if ([ "$1" == "-d" ] && ! grep -iq "#SET_REPR_MIRRORS" "$mirror") || + ([ "$1" != "-d" ] && grep -iq "#SET_REPR_MIRRORS" "$mirror"); then + continue + fi + + # Enable or disable the reproducible mirrors $SUDO sed -i "$expression" "$mirror" + + # Enable or disable the none reproducible mirrors + if [ "$MIRROR_SNAPSHOT" == y ]; then + $SUDO sed -ri "$expression2" "$mirror" + fi + + # Add or remove the SET_REPR_MIRRORS flag + $SUDO sed -i "$expression3" "$mirror" done } diff --git a/src/sonic-build-hooks/scripts/collect_version_files b/src/sonic-build-hooks/scripts/collect_version_files index a4b33eeaa897..3fcc99a87106 100755 --- a/src/sonic-build-hooks/scripts/collect_version_files +++ b/src/sonic-build-hooks/scripts/collect_version_files @@ -18,6 +18,13 @@ dpkg-query -W -f '${Package}==${Version}\n' >> "${TARGET_PATH}/versions-deb-${DI ## Add the the packages purged [ -f $POST_VERSION_PATH/purge-versions-deb ] && cat $POST_VERSION_PATH/purge-versions-deb >> "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" +## Add mirror versions +while read -r line; do + mirror=$(echo "$line" | sed "s/.*\///" | sed "s/_InRelease.*//") + date=$(date --date="$(echo "$line" | cut -d: -f3-)" +%Y-%m-%dT%H:%M:%SZ) + echo "$mirror==$date" >> ${TARGET_PATH}/versions-mirror +done < <(grep Date: /var/lib/apt/lists/*_InRelease 2>/dev/null) + ## Print the unique and sorted result sort -u "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" if [ -e "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}" ]; then @@ -26,5 +33,8 @@ fi if [ -e "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" ]; then sort -u "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" fi +if [ -e "${TARGET_PATH}/versions-mirror" ]; then + sort -u "${TARGET_PATH}/versions-mirror" -o "${TARGET_PATH}/versions-mirror" +fi exit 0