[v1.x] Migrate to use ECR as docker cache instead of dockerhub

ci/Jenkinsfile_docker_cache

@@ -37,7 +37,7 @@ core_logic: {
       ws('workspace/docker_cache') {
         timeout(time: total_timeout, unit: 'MINUTES') {
           utils.init_git()
-          sh "ci/docker_cache.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}"
+          sh "ci/docker_cache.py --docker-registry ${env.DOCKER_ECR_REGISTRY}"
         }
       }
     }

ci/Jenkinsfile_utils.groovy

@@ -160,7 +160,7 @@ def collect_test_results_windows(original_file_name, new_file_name) {
 
 
 def docker_run(platform, function_name, use_nvidia, shared_mem = '500m', env_vars = "") {
-  def command = "ci/build.py %ENV_VARS% --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
+  def command = "ci/build.py %ENV_VARS% --docker-registry ${env.DOCKER_ECR_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
   command = command.replaceAll('%ENV_VARS%', env_vars.length() > 0 ? "-e ${env_vars}" : '')
   command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '')
   command = command.replaceAll('%PLATFORM%', platform)

ci/build.py

@@ -27,6 +27,8 @@
 
 import argparse
 import glob
+import hashlib
+import os
 import pprint
 import re
 import shutil
@@ -52,13 +54,41 @@ def get_platforms(path: str = get_dockerfiles_path()) -> List[str]:
     platforms = list(map(lambda x: os.path.split(x)[1], sorted(files)))
     return platforms
 
+def _find_copied_files(dockerfile):
+    """
+    Creates a list of files copied into given dockerfile.
+    """
+    copied_files = []
+    basedir = os.path.dirname(dockerfile)
+    with open(dockerfile, "r") as f:
+        for line in f.readlines():
+            if line.startswith("COPY "):
+                copied_files.append(os.path.join(basedir, line.split(" ")[1]))
+    return copied_files
+
+def _hash_file(ctx, filename):
+    """
+    Add contents of passed file into passed hash context.
+    """
+    bufsiz = 16384
+    with open(filename,"rb") as f:
+        while True:
+            d = f.read(bufsiz)
+            if not d:
+                break
+            ctx.update(d)
 
 def get_docker_tag(platform: str, registry: str) -> str:
     """:return: docker tag to be used for the container"""
     platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform)
     if not registry:
         registry = "mxnet_local"
-    return "{0}/{1}".format(registry, platform)
+    dockerfile = get_dockerfile(platform)
+    sha256 = hashlib.sha256()
+    _hash_file(sha256, dockerfile)
+    for f in _find_copied_files(dockerfile):
+        _hash_file(sha256, f)
+    return "{0}:{1}-{2}".format(registry, platform, sha256.hexdigest()[:12])
 
 
 def get_dockerfile(platform: str, path=get_dockerfiles_path()) -> str:
@@ -265,6 +295,19 @@ def load_docker_cache(tag, docker_registry) -> None:
     else:
         logging.info('Distributed docker cache disabled')
 
+def push_docker_cache(registry, tag, image_id) -> None:
+    """Uploads tagged container to given docker registry"""
+    if registry:
+        # noinspection PyBroadException
+        try:
+            import docker_cache
+            logging.info('Docker cache upload is enabled to registry %s', registry)
+            docker_cache._upload_image(registry, tag, image_id)
+        except Exception:
+            logging.exception('Unable to push image to Docker cache...')
+    else:
+        logging.info('Distributed docker cache disabled')
+
 
 def log_environment():
     instance_info = ec2_instance_info()
@@ -406,7 +449,8 @@ def main() -> int:
             tag = get_docker_tag(platform=platform, registry=args.docker_registry)
             load_docker_cache(tag=tag, docker_registry=args.docker_registry)
             build_docker(platform, registry=args.docker_registry,
-                         num_retries=args.docker_build_retries, no_cache=args.no_cache)
+                         num_retries=args.docker_build_retries, no_cache=args.no_cache,
+                         cache_intermediate=args.cache_intermediate)
             if args.build_only:
                 continue
             shutil.rmtree(buildir(), ignore_errors=True)

ci/docker_cache.py

@@ -84,7 +84,7 @@ def _build_save_container(platform, registry, load_cache) -> Optional[str]:
     logging.debug('Building %s as %s', platform, docker_tag)
     try:
         # Increase the number of retries for building the cache.
-        image_id = build_util.build_docker(docker_binary='docker', platform=platform, registry=registry, num_retries=10, no_cache=False)
+        image_id = build_util.build_docker(platform=platform, registry=registry, num_retries=10, no_cache=False)
         logging.info('Built %s as %s', docker_tag, image_id)
 
         # Push cache to registry
@@ -96,6 +96,14 @@ def _build_save_container(platform, registry, load_cache) -> Optional[str]:
         # Error handling is done by returning the errorous platform name. This is necessary due to
         # Parallel being unable to handle exceptions
 
+def _ecr_login(registry):
+    """
+    Use the AWS CLI to get credentials to login to ECR.
+    """
+    # extract region from registry
+    region = registry.split(".")[3]
+    logging.info("Logging into ECR region %s using aws-cli..", region)
+    os.system("$(aws ecr get-login --region "+region+" --no-include-email)")
 
 def _upload_image(registry, docker_tag, image_id) -> None:
     """
@@ -105,6 +113,10 @@ def _upload_image(registry, docker_tag, image_id) -> None:
     :param image_id: Image id
     :return: None
     """
+
+    if "dkr.ecr" in registry:
+        _ecr_login(registry)
+
     # We don't have to retag the image since it is already in the right format
     logging.info('Uploading %s (%s) to %s', docker_tag, image_id, registry)
     push_cmd = ['docker', 'push', docker_tag]
@@ -125,6 +137,9 @@ def load_docker_cache(registry, docker_tag) -> None:
         return
     assert docker_tag
 
+    if "dkr.ecr" in registry:
+        _ecr_login(registry)
+
     logging.info('Loading Docker cache for %s from %s', docker_tag, registry)
     pull_cmd = ['docker', 'pull', docker_tag]