Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Add timeout/retry logic to docker cache download #13573

Merged
merged 6 commits into from
Dec 21, 2018
Merged

Add timeout/retry logic to docker cache download #13573

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4a87b84
Added timeout/retry (linear backoff) to docker cache download
Dec 6, 2018
1b819ce
Units changed, as time.sleep takes seconds as argument
Dec 7, 2018
f9c59a6
Improved error handling
Dec 7, 2018
31472a6
Using retry decorator
Dec 10, 2018
b25984a
Added retry decorator to _login_dockerhub method
Dec 10, 2018
e7df3ee
Fixed wrong import
Dec 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 25 additions & 26 deletions ci/docker_cache.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,15 @@
import sys
import subprocess
import json
import time
from typing import *
import build as build_util
from util import retry

DOCKERHUB_LOGIN_NUM_RETRIES = 5
DOCKERHUB_RETRY_SECONDS = 5
DOCKER_CACHE_NUM_RETRIES = 3
DOCKER_CACHE_TIMEOUT_MINS = 5


def build_save_containers(platforms, registry, load_cache) -> int:
"""
Expand Down Expand Up @@ -107,37 +110,28 @@ def _upload_image(registry, docker_tag, image_id) -> None:
subprocess.check_call(push_cmd)


@retry(target_exception=subprocess.CalledProcessError, tries=DOCKERHUB_LOGIN_NUM_RETRIES,
delay_s=DOCKERHUB_RETRY_SECONDS)
def _login_dockerhub():
"""
Login to the Docker Hub account
:return: None
"""
dockerhub_credentials = _get_dockerhub_credentials()

for i in range(DOCKERHUB_LOGIN_NUM_RETRIES):
logging.info('Logging in to DockerHub')
# We use password-stdin instead of --password to avoid leaking passwords in case of an error.
# This method will produce the following output:
# > WARNING! Your password will be stored unencrypted in /home/jenkins_slave/.docker/config.json.
# > Configure a credential helper to remove this warning. See
# > https://docs.docker.com/engine/reference/commandline/login/#credentials-store
# Since we consider the restricted slaves a secure environment, that's fine. Also, using this will require
# third party applications which would need a review first as well.
p = subprocess.run(['docker', 'login', '--username', dockerhub_credentials['username'], '--password-stdin'],
stdout=subprocess.PIPE, input=str.encode(dockerhub_credentials['password']))
logging.info(p.stdout)
if p.returncode != 0:
logging.error('Error logging in to DockerHub')
logging.error(p.stderr)

# Linear backoff
time.sleep(1000 * DOCKERHUB_RETRY_SECONDS * (i + 1))
else:
logging.info('Successfully logged in to DockerHub')
break
else:
logging.error('DockerHub login not possible after %d retries, aborting', DOCKERHUB_LOGIN_NUM_RETRIES)
raise Exception('Unable to log in to DockerHub')
logging.info('Logging in to DockerHub')
# We use password-stdin instead of --password to avoid leaking passwords in case of an error.
# This method will produce the following output:
# > WARNING! Your password will be stored unencrypted in /home/jenkins_slave/.docker/config.json.
# > Configure a credential helper to remove this warning. See
# > https://docs.docker.com/engine/reference/commandline/login/#credentials-store
# Since we consider the restricted slaves a secure environment, that's fine. Also, using this will require
# third party applications which would need a review first as well.
p = subprocess.run(['docker', 'login', '--username', dockerhub_credentials['username'], '--password-stdin'],
stdout=subprocess.PIPE, input=str.encode(dockerhub_credentials['password']))
logging.info(p.stdout)
logging.info('Successfully logged in to DockerHub')


def _logout_dockerhub():
"""
Expand All @@ -149,6 +143,8 @@ def _logout_dockerhub():
logging.info('Successfully logged out of DockerHub')


@retry(target_exception=subprocess.TimeoutExpired, tries=DOCKER_CACHE_NUM_RETRIES,
delay_s=DOCKERHUB_RETRY_SECONDS)
def load_docker_cache(registry, docker_tag) -> None:
"""
Load the precompiled docker cache from the registry
Expand All @@ -163,7 +159,10 @@ def load_docker_cache(registry, docker_tag) -> None:

logging.info('Loading Docker cache for %s from %s', docker_tag, registry)
pull_cmd = ['docker', 'pull', docker_tag]
subprocess.call(pull_cmd) # Don't throw an error if the image does not exist

# Don't throw an error if the image does not exist
subprocess.run(pull_cmd, timeout=DOCKER_CACHE_TIMEOUT_MINS*60)
logging.info('Successfully pulled docker cache')


def delete_local_docker_cache(docker_tag):
Expand Down