From 3b76515aa35c5605b2eb20bcc18e8f856ad5e533 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Fri, 12 Feb 2021 05:19:56 +0000 Subject: [PATCH 1/2] Add random sleep only, since retry attempts are already implemented. --- ci/safe_docker_run.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/safe_docker_run.py b/ci/safe_docker_run.py index 97ece4aecd2f..738ab342f5b7 100755 --- a/ci/safe_docker_run.py +++ b/ci/safe_docker_run.py @@ -26,8 +26,10 @@ import atexit import logging import os +import random import signal import sys +import time from functools import reduce from itertools import chain from typing import Dict, Any @@ -117,6 +119,9 @@ def run(self, *args, **kwargs) -> int: ret = 0 try: # Race condition: + # add a random sleep to (a) give docker time to flush disk buffer after pulling image + # and (b) minimize race conditions between jenkins runs on same host + time.sleep(random.randint(5,20)) # If the call to docker_client.containers.run is interrupted, it is possible that # the container won't be cleaned up. We avoid this by temporarily masking the signals. signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM}) From 6406b8324d497656784ad02f7586b6f45fdcf146 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Fri, 12 Feb 2021 05:31:43 +0000 Subject: [PATCH 2/2] Reduce random sleep to 2-10 sec. --- ci/safe_docker_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/safe_docker_run.py b/ci/safe_docker_run.py index 738ab342f5b7..9c90c2aaada9 100755 --- a/ci/safe_docker_run.py +++ b/ci/safe_docker_run.py @@ -121,7 +121,7 @@ def run(self, *args, **kwargs) -> int: # Race condition: # add a random sleep to (a) give docker time to flush disk buffer after pulling image # and (b) minimize race conditions between jenkins runs on same host - time.sleep(random.randint(5,20)) + time.sleep(random.randint(2,10)) # If the call to docker_client.containers.run is interrupted, it is possible that # the container won't be cleaned up. We avoid this by temporarily masking the signals. signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM})