[build][packaging] Add resilience when docker build (#22050) (#22082)

Author: v1v

dev-tools/mage/dockerbuilder.go

@@ -26,6 +26,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/magefile/mage/sh"
 	"github.com/pkg/errors"
@@ -71,7 +72,13 @@ func (b *dockerBuilder) Build() error {
 
 	tag, err := b.dockerBuild()
 	if err != nil {
-		return errors.Wrap(err, "failed to build docker")
+		fmt.Println(">> Building docker images again (after 10 seconds)")
+		// This sleep is to avoid hitting the docker build issues when resources are not available.
+		time.Sleep(10)
+		tag, err = b.dockerBuild()
+		if err != nil {
+			return errors.Wrap(err, "failed to build docker")
+		}
 	}
 
 	if err := b.dockerSave(tag); err != nil {

x-pack/elastic-agent/magefile.go

@@ -507,8 +507,13 @@ func runAgent(env map[string]string) error {
 		}
 
 		// build docker image
-		if err := sh.Run("docker", "build", "-t", tag, "."); err != nil {
-			return err
+		if err := dockerBuild(tag); err != nil {
+			fmt.Println(">> Building docker images again (after 10 seconds)")
+			// This sleep is to avoid hitting the docker build issues when resources are not available.
+			time.Sleep(10)
+			if err := dockerBuild(tag); err != nil {
+				return err
+			}
 		}
 	}
 
@@ -612,6 +617,10 @@ func copyAll(from, to string) error {
 	})
 }
 
+func dockerBuild(tag string) error {
+	return sh.Run("docker", "build", "-t", tag, ".")
+}
+
 func dockerTag() string {
 	const commitLen = 7
 	tagBase := "elastic-agent"