rebase

Author: driazati

Jenkinsfile

@@ -83,8 +83,10 @@ tvm_multilib_tsim = 'build/libvta_tsim.so, ' +
 
 // command to start a docker container
 docker_run = 'docker/bash.sh'
+docker_build = 'docker/build.sh'
 // timeout in minutes
 max_time = 240
+rebuild_docker_images = false
 
 def per_exec_ws(folder) {
   return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
@@ -200,11 +202,21 @@ stage('Sanity Check') {
         init_git()
         is_docs_only_build = sh (
           returnStatus: true,
-          script: './tests/scripts/git_change_docs.sh',
+          script: './tests/scripts/git_change_files.sh docs/',
           label: 'Check for docs only changes',
         )
         skip_ci = should_skip_ci(env.CHANGE_ID)
         skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
+        rebuild_docker_images = sh (
+          returnStatus: true,
+          script: './tests/scripts/git_change_files.sh docker/',
+          label: 'Check for docs only changes',
+        )
+        if (rebuild_docker_images) {
+          // Exit before linting so we can use the newly created Docker images
+          // to run the lint
+          return
+        }
         sh (
           script: "${docker_run} ${ci_lint}  ./tests/scripts/task_lint.sh",
           label: 'Run lint',
@@ -214,6 +226,104 @@ stage('Sanity Check') {
   }
 }
 
+def build_image(image_name) {
+  hash = sh(
+    returnStdout: true,
+    script: 'git log -1 --format=\'%h\''
+  ).trim()
+  hash = "${env.BRANCH_NAME}-${hash}"
+  sh(
+    script: "${docker_build} ${image_name} --spec ${image_name}:${hash}",
+    label: 'Building docker image'
+  )
+  sh(
+    script: "docker rmi ${image_name}:${hash}",
+    label: 'Removing docker image'
+  )
+  sh "echo NYI: Uploading docker image to registry..."
+}
+
+if (rebuild_docker_images) {
+  stage('Docker Image Build') {
+    // TODO in a follow up PR: Upload to ECR, find tag and use in
+    // subsequent builds
+    parallel 'ci-lint': {
+      node('CPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_lint')
+        }
+      }
+    }, 'ci-cpu': {
+      node('CPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_cpu')
+        }
+      }
+    }, 'ci-gpu': {
+      node('GPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_gpu')
+        }
+      }
+    }, 'ci-qemu': {
+      node('CPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_qemu')
+        }
+      }
+    }, 'ci-i386': {
+      node('CPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_i386')
+        }
+      }
+    }, 'ci-arm': {
+      node('ARM') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_arm')
+        }
+      }
+    }, 'ci-wasm': {
+      node('CPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_wasm')
+        }
+      }
+    }, 'ci-hexagon': {
+      node('CPU') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          init_git()
+          build_image('ci_hexagon')
+        }
+      }
+    }
+  }
+  // // TODO: Once we are able to use the built images, enable this step
+  // // If the docker images changed, we need to run the image build before the lint
+  // // can run since it requires a base docker image. Most of the time the images
+  // // aren't build though so it's faster to use the same node that checks for
+  // // docker changes to run the lint in the usual case.
+  // stage('Sanity Check (re-run)') {
+  //   timeout(time: max_time, unit: 'MINUTES') {
+  //     node('CPU') {
+  //       ws(per_exec_ws('tvm/sanity')) {
+  //         init_git()
+  //         sh (
+  //           script: "${docker_run} ${ci_lint}  ./tests/scripts/task_lint.sh",
+  //           label: 'Run lint',
+  //         )
+  //       }
+  //     }
+  //   }
+  // }
+}
 
 // Run make. First try to do an incremental make from a previous workspace in hope to
 // accelerate the compilation. If something is wrong, clean the workspace and then

docker/build.sh

@@ -24,6 +24,7 @@
 #                [--dockerfile <DOCKERFILE_PATH>] [-it]
 #                [--net=host] [--cache-from <IMAGE_NAME>]
 #                [--name CONTAINER_NAME] [--context-path <CONTEXT_PATH>]
+#                [--spec DOCKER_IMAGE_SPEC]
 #                [<COMMAND>]
 #
 # CONTAINER_TYPE: Type of the docker container used the run the build,
@@ -36,6 +37,9 @@
 #                  this optional value is not supplied (via the --dockerfile
 #                  flag), will use Dockerfile.CONTAINER_TYPE in default
 #
+# DOCKER_IMAGE_SPEC: Override the default logic to determine the image name and
+#                    tag
+#
 # IMAGE_NAME: An image to be as a source for cached layers when building the
 #             Docker image requested.
 #
@@ -73,6 +77,11 @@ if [[ "$1" == "-it" ]]; then
     shift 1
 fi
 
+if [[ "$1" == "--spec" ]]; then
+    OVERRIDE_IMAGE_SPEC="$2"
+    shift 2
+fi
+
 if [[ "$1" == "--net=host" ]]; then
     CI_DOCKER_EXTRA_PARAMS+=('--net=host')
     CI_DOCKER_BUILD_EXTRA_PARAMS+=("--network=host")
@@ -162,6 +171,10 @@ DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]')
 # Compose the full image spec with "name:tag" e.g. "tvm.ci_cpu:v0.03"
 DOCKER_IMG_SPEC="${DOCKER_IMG_NAME}:${DOCKER_IMAGE_TAG}"
 
+if [[ -n ${OVERRIDE_IMAGE_SPEC+x} ]]; then
+    DOCKER_IMG_SPEC="$OVERRIDE_IMAGE_SPEC"
+fi
+
 # Print arguments.
 echo "WORKSPACE: ${WORKSPACE}"
 echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}"

tests/scripts/git_change_docs.sh renamed to tests/scripts/git_change_files.sh

@@ -20,19 +20,20 @@
 set -eux
 
 FOUND_ONE_FILE=0
-SAW_NON_DOC_CHANGES=0
+SAW_NON_RELEVANT_CHANGES=0
+FOLDER_WITH_CHANGES="$1"
 
 changed_files=$(git diff --no-commit-id --name-only -r origin/main)
 
 for file in $changed_files; do
     FOUND_ONE_FILE=1
-    if ! grep -q "docs/" <<< "$file"; then
-        SAW_NON_DOC_CHANGES=1
+    if ! grep -q "$FOLDER_WITH_CHANGES" <<< "$file"; then
+        SAW_NON_RELEVANT_CHANGES=1
         break
     fi
 done
 
-if [ ${FOUND_ONE_FILE} -eq 0 ] || [ ${SAW_NON_DOC_CHANGES} -eq 1 ]; then
+if [ ${FOUND_ONE_FILE} -eq 0 ] || [ ${SAW_NON_RELEVANT_CHANGES} -eq 1 ]; then
     exit 0
 else
     exit 1