[v1.x] Address CI failures with docker timeouts (v2) (apache#19890)

* Add random sleep only, since retry attempts are already implemented. * Reduce random sleep to 2-10 sec. Co-authored-by: Joe Evans <[email protected]>
josephevans · Feb 24, 2021 · e35eb2e · e35eb2e
1 parent e81d2b9
commit e35eb2e
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/ci/safe_docker_run.py b/ci/safe_docker_run.py
@@ -26,8 +26,10 @@
 import atexit
 import logging
 import os
+import random
 import signal
 import sys
+import time
 from functools import reduce
 from itertools import chain
 from typing import Dict, Any
@@ -117,6 +119,9 @@ def run(self, *args, **kwargs) -> int:
         ret = 0
         try:
             # Race condition:
+            # add a random sleep to (a) give docker time to flush disk buffer after pulling image
+            # and (b) minimize race conditions between jenkins runs on same host
+            time.sleep(random.randint(2,10))
             # If the call to docker_client.containers.run is interrupted, it is possible that
             # the container won't be cleaned up. We avoid this by temporarily masking the signals.
             signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM})