[v1.x] Address CI failures with docker timeouts (v2) (#19890)

* Add random sleep only, since retry attempts are already implemented. * Reduce random sleep to 2-10 sec. Co-authored-by: Joe Evans <[email protected]>
apache · Feb 12, 2021 · b5b6743 · b5b6743
1 parent 80ccb4b
commit b5b6743
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/ci/safe_docker_run.py b/ci/safe_docker_run.py
@@ -26,8 +26,10 @@
 import atexit
 import logging
 import os
+import random
 import signal
 import sys
+import time
 from functools import reduce
 from itertools import chain
 from typing import Dict, Any
@@ -117,6 +119,9 @@ def run(self, *args, **kwargs) -> int:
         ret = 0
         try:
             # Race condition:
+            # add a random sleep to (a) give docker time to flush disk buffer after pulling image
+            # and (b) minimize race conditions between jenkins runs on same host
+            time.sleep(random.randint(2,10))
             # If the call to docker_client.containers.run is interrupted, it is possible that
             # the container won't be cleaned up. We avoid this by temporarily masking the signals.
             signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM})