draft

.azure-pipelines/template-variables.yml

@@ -2,3 +2,4 @@ variables:
   DEFAULT_CONTAINER_REGISTRY: 'publicmirror.azurecr.io'
   COMMON_LIB_BUILD_ENVS: 'bullseye'
   SONIC_SLAVE_DOCKER_DRIVER: 'overlay2'
+  SONIC_BUILD_RETRY_COUNT: 3

Makefile

@@ -33,6 +33,8 @@ PLATFORM_PATH := platform/$(if $(PLATFORM),$(PLATFORM),$(CONFIGURED_PLATFORM))
 PLATFORM_CHECKOUT := platform/checkout
 PLATFORM_CHECKOUT_FILE := $(PLATFORM_CHECKOUT)/$(PLATFORM).ini
 PLATFORM_CHECKOUT_CMD := $(shell if [ -f $(PLATFORM_CHECKOUT_FILE) ]; then PLATFORM_PATH=$(PLATFORM_PATH) j2 $(PLATFORM_CHECKOUT)/template.j2 $(PLATFORM_CHECKOUT_FILE); fi)
+RUN_WITH_RETRY := $(shell [ $(SONIC_BUILD_RETRY_COUNT) -ge 1 ] && echo ". scripts/functions.sh && SONIC_BUILD_RETRY_INTERVAL=$(SONIC_BUILD_RETRY_INTERVAL) SONIC_BUILD_RETRY_COUNT=$(SONIC_BUILD_RETRY_COUNT) run_with_retry" )
+MAKE := $(RUN_WITH_RETRY) $(MAKE)
 
 %::
 	@echo "+++ --- Making $@ --- +++"

rules/config

@@ -256,3 +256,7 @@ ENABLE_FIPS ?= n
 
 # SONIC_SLAVE_DOCKER_DRIVER - set the sonic slave docker storage driver
 SONIC_SLAVE_DOCKER_DRIVER ?= vfs
+
+# Add retry when build failed
+SONIC_BUILD_RETRY_COUNT ?= 0
+SONIC_BUILD_RETRY_INTERVAL ?= 600

scripts/functions.sh

@@ -0,0 +1,10 @@
+run_with_retry(){
+    [ $SONIC_BUILD_RETRY_COUNT -lt 0 ] && SONIC_BUILD_RETRY_COUNT=0
+    [[ "$*" == "" ]] && { echo "run_with_retry: input command can't be empty." 1>&2;exit 1; }
+    for ((i=0; i<=$SONIC_BUILD_RETRY_COUNT; i++))
+    do
+        [[ $i != 0 ]] && { echo "================================================================================================" 1>&2; echo "Waiting $SONIC_BUILD_RETRY_INTERVAL to run again, $i/$SONIC_BUILD_RETRY_COUNT" 1>&2; echo "================================================================================================" 1>&2; sleep $SONIC_BUILD_RETRY_INTERVAL; }
+        "$@" && break
+    done
+}
+