[ci] Rebuild docker images on docker changes

Jenkinsfile

@@ -83,8 +83,10 @@ tvm_multilib_tsim = 'build/libvta_tsim.so, ' +
 
 // command to start a docker container
 docker_run = 'docker/bash.sh'
+docker_build = 'docker/build.sh'
 // timeout in minutes
 max_time = 240
+rebuild_docker_images = false
 
 def per_exec_ws(folder) {
   return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
@@ -205,6 +207,16 @@ stage('Sanity Check') {
         )
         skip_ci = should_skip_ci(env.CHANGE_ID)
         skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
+        rebuild_docker_images = sh (
+          returnStatus: true,
+          script: './tests/scripts/git_change_docker.sh',
+          label: 'Check for any docker changes',
+        )
+        if (rebuild_docker_images) {
+          // Exit before linting so we can use the newly created Docker images
+          // to run the lint
+          return
+        }
         sh (
           script: "${docker_run} ${ci_lint}  ./tests/scripts/task_lint.sh",
           label: 'Run lint',
@@ -214,6 +226,104 @@ stage('Sanity Check') {
   }
 }
 
+def build_image(image_name) {
+  hash = sh(
+    returnStdout: true,
+    script: 'git log -1 --format=\'%h\''
+  ).trim()
+  def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}"
+  sh(
+    script: "${docker_build} ${image_name} --spec ${full_name}",
+    label: 'Building docker image'
+  )
+  sh(
+    script: "docker rmi ${full_name}",
+    label: 'Removing docker image'
+  )
+  sh "echo NYI: Uploading docker image to registry..."
+}
+
+if (rebuild_docker_images) {
+  stage('Docker Image Build') {
+    // TODO in a follow up PR: Upload to ECR, find tag and use in
+    // subsequent builds
+    parallel 'ci-lint': {
+      node('CPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_lint')
+        }
+      }
+    }, 'ci-cpu': {
+      node('CPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_cpu')
+        }
+      }
+    }, 'ci-gpu': {
+      node('GPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_gpu')
+        }
+      }
+    }, 'ci-qemu': {
+      node('CPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_qemu')
+        }
+      }
+    }, 'ci-i386': {
+      node('CPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_i386')
+        }
+      }
+    }, 'ci-arm': {
+      node('ARM') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_arm')
+        }
+      }
+    }, 'ci-wasm': {
+      node('CPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_wasm')
+        }
+      }
+    }, 'ci-hexagon': {
+      node('CPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_hexagon')
+        }
+      }
+    }
+  }
+  // // TODO: Once we are able to use the built images, enable this step
+  // // If the docker images changed, we need to run the image build before the lint
+  // // can run since it requires a base docker image. Most of the time the images
+  // // aren't build though so it's faster to use the same node that checks for
+  // // docker changes to run the lint in the usual case.
+  // stage('Sanity Check (re-run)') {
+  //   timeout(time: max_time, unit: 'MINUTES') {
+  //     node('CPU') {
+  //       ws(per_exec_ws('tvm/sanity')) {
+  //         init_git()
+  //         sh (
+  //           script: "${docker_run} ${ci_lint}  ./tests/scripts/task_lint.sh",
+  //           label: 'Run lint',
+  //         )
+  //       }
+  //     }
+  //   }
+  // }
+}
 
 // Run make. First try to do an incremental make from a previous workspace in hope to
 // accelerate the compilation. If something is wrong, clean the workspace and then

docker/build.sh

@@ -24,6 +24,7 @@
 #                [--dockerfile <DOCKERFILE_PATH>] [-it]
 #                [--net=host] [--cache-from <IMAGE_NAME>]
 #                [--name CONTAINER_NAME] [--context-path <CONTEXT_PATH>]
+#                [--spec DOCKER_IMAGE_SPEC]
 #                [<COMMAND>]
 #
 # CONTAINER_TYPE: Type of the docker container used the run the build,
@@ -36,6 +37,9 @@
 #                  this optional value is not supplied (via the --dockerfile
 #                  flag), will use Dockerfile.CONTAINER_TYPE in default
 #
+# DOCKER_IMAGE_SPEC: Override the default logic to determine the image name and
+#                    tag
+#
 # IMAGE_NAME: An image to be as a source for cached layers when building the
 #             Docker image requested.
 #
@@ -73,6 +77,11 @@ if [[ "$1" == "-it" ]]; then
     shift 1
 fi
 
+if [[ "$1" == "--spec" ]]; then
+    OVERRIDE_IMAGE_SPEC="$2"
+    shift 2
+fi
+
 if [[ "$1" == "--net=host" ]]; then
     CI_DOCKER_EXTRA_PARAMS+=('--net=host')
     CI_DOCKER_BUILD_EXTRA_PARAMS+=("--network=host")
@@ -162,6 +171,10 @@ DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]')
 # Compose the full image spec with "name:tag" e.g. "tvm.ci_cpu:v0.03"
 DOCKER_IMG_SPEC="${DOCKER_IMG_NAME}:${DOCKER_IMAGE_TAG}"
 
+if [[ -n ${OVERRIDE_IMAGE_SPEC+x} ]]; then
+    DOCKER_IMG_SPEC="$OVERRIDE_IMAGE_SPEC"
+fi
+
 # Print arguments.
 echo "WORKSPACE: ${WORKSPACE}"
 echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}"

tests/scripts/git_change_docker.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eux
+
+changed_files=$(git diff --no-commit-id --name-only -r origin/main)
+
+for file in $changed_files; do
+    echo "Checking $file"
+    if grep -q "docker/" <<< "$file"; then
+        exit 1
+    fi
+done
+
+# No docker changes
+exit 0