Add timeout/retry logic to docker cache download (#13573)

* Added timeout/retry (linear backoff) to docker cache download

* Units changed, as time.sleep takes seconds as argument

* Improved error handling

* Using retry decorator

* Added retry decorator to _login_dockerhub method

* Fixed wrong import

ci/docker_cache.py

@@ -30,12 +30,15 @@
 import sys
 import subprocess
 import json
-import time
 from typing import *
 import build as build_util
+from util import retry
 
 DOCKERHUB_LOGIN_NUM_RETRIES = 5
 DOCKERHUB_RETRY_SECONDS = 5
+DOCKER_CACHE_NUM_RETRIES = 3
+DOCKER_CACHE_TIMEOUT_MINS = 5
+
 
 def build_save_containers(platforms, registry, load_cache) -> int:
     """
@@ -107,37 +110,28 @@ def _upload_image(registry, docker_tag, image_id) -> None:
     subprocess.check_call(push_cmd)
 
 
+@retry(target_exception=subprocess.CalledProcessError, tries=DOCKERHUB_LOGIN_NUM_RETRIES,
+       delay_s=DOCKERHUB_RETRY_SECONDS)
 def _login_dockerhub():
     """
     Login to the Docker Hub account
     :return: None
     """
     dockerhub_credentials = _get_dockerhub_credentials()
 
-    for i in range(DOCKERHUB_LOGIN_NUM_RETRIES):
-        logging.info('Logging in to DockerHub')
-        # We use password-stdin instead of --password to avoid leaking passwords in case of an error.
-        # This method will produce the following output:
-        # > WARNING! Your password will be stored unencrypted in /home/jenkins_slave/.docker/config.json.
-        # > Configure a credential helper to remove this warning. See
-        # > https://docs.docker.com/engine/reference/commandline/login/#credentials-store
-        # Since we consider the restricted slaves a secure environment, that's fine. Also, using this will require
-        # third party applications which would need a review first as well.
-        p = subprocess.run(['docker', 'login', '--username', dockerhub_credentials['username'], '--password-stdin'],
-                           stdout=subprocess.PIPE, input=str.encode(dockerhub_credentials['password']))
-        logging.info(p.stdout)
-        if p.returncode != 0:
-            logging.error('Error logging in to DockerHub')
-            logging.error(p.stderr)
-
-            # Linear backoff
-            time.sleep(1000 * DOCKERHUB_RETRY_SECONDS * (i + 1))
-        else:
-            logging.info('Successfully logged in to DockerHub')
-            break
-    else:
-        logging.error('DockerHub login not possible after %d retries, aborting', DOCKERHUB_LOGIN_NUM_RETRIES)
-        raise Exception('Unable to log in to DockerHub')
+    logging.info('Logging in to DockerHub')
+    # We use password-stdin instead of --password to avoid leaking passwords in case of an error.
+    # This method will produce the following output:
+    # > WARNING! Your password will be stored unencrypted in /home/jenkins_slave/.docker/config.json.
+    # > Configure a credential helper to remove this warning. See
+    # > https://docs.docker.com/engine/reference/commandline/login/#credentials-store
+    # Since we consider the restricted slaves a secure environment, that's fine. Also, using this will require
+    # third party applications which would need a review first as well.
+    p = subprocess.run(['docker', 'login', '--username', dockerhub_credentials['username'], '--password-stdin'],
+                       stdout=subprocess.PIPE, input=str.encode(dockerhub_credentials['password']))
+    logging.info(p.stdout)
+    logging.info('Successfully logged in to DockerHub')
+
 
 def _logout_dockerhub():
     """
@@ -149,6 +143,8 @@ def _logout_dockerhub():
     logging.info('Successfully logged out of DockerHub')
 
 
+@retry(target_exception=subprocess.TimeoutExpired, tries=DOCKER_CACHE_NUM_RETRIES,
+       delay_s=DOCKERHUB_RETRY_SECONDS)
 def load_docker_cache(registry, docker_tag) -> None:
     """
     Load the precompiled docker cache from the registry
@@ -163,7 +159,10 @@ def load_docker_cache(registry, docker_tag) -> None:
 
     logging.info('Loading Docker cache for %s from %s', docker_tag, registry)
     pull_cmd = ['docker', 'pull', docker_tag]
-    subprocess.call(pull_cmd)  # Don't throw an error if the image does not exist
+
+    # Don't throw an error if the image does not exist
+    subprocess.run(pull_cmd, timeout=DOCKER_CACHE_TIMEOUT_MINS*60)
+    logging.info('Successfully pulled docker cache')
 
 
 def delete_local_docker_cache(docker_tag):