diff --git a/Jenkinsfile b/Jenkinsfile index 44f7b591cfaf..19320ba24faf 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -83,8 +83,10 @@ tvm_multilib_tsim = 'build/libvta_tsim.so, ' + // command to start a docker container docker_run = 'docker/bash.sh' +docker_build = 'docker/build.sh' // timeout in minutes max_time = 240 +rebuild_docker_images = false def per_exec_ws(folder) { return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder @@ -205,6 +207,16 @@ stage('Sanity Check') { ) skip_ci = should_skip_ci(env.CHANGE_ID) skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) + rebuild_docker_images = sh ( + returnStatus: true, + script: './tests/scripts/git_change_docker.sh', + label: 'Check for any docker changes', + ) + if (rebuild_docker_images) { + // Exit before linting so we can use the newly created Docker images + // to run the lint + return + } sh ( script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", label: 'Run lint', @@ -214,6 +226,104 @@ stage('Sanity Check') { } } +def build_image(image_name) { + hash = sh( + returnStdout: true, + script: 'git log -1 --format=\'%h\'' + ).trim() + def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}" + sh( + script: "${docker_build} ${image_name} --spec ${full_name}", + label: 'Building docker image' + ) + sh( + script: "docker rmi ${full_name}", + label: 'Removing docker image' + ) + sh "echo NYI: Uploading docker image to registry..." +} + +if (rebuild_docker_images) { + stage('Docker Image Build') { + // TODO in a follow up PR: Upload to ECR, find tag and use in + // subsequent builds + parallel 'ci-lint': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_lint') + } + } + }, 'ci-cpu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_cpu') + } + } + }, 'ci-gpu': { + node('GPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_gpu') + } + } + }, 'ci-qemu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_qemu') + } + } + }, 'ci-i386': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_i386') + } + } + }, 'ci-arm': { + node('ARM') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_arm') + } + } + }, 'ci-wasm': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_wasm') + } + } + }, 'ci-hexagon': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_hexagon') + } + } + } + } + // // TODO: Once we are able to use the built images, enable this step + // // If the docker images changed, we need to run the image build before the lint + // // can run since it requires a base docker image. Most of the time the images + // // aren't build though so it's faster to use the same node that checks for + // // docker changes to run the lint in the usual case. + // stage('Sanity Check (re-run)') { + // timeout(time: max_time, unit: 'MINUTES') { + // node('CPU') { + // ws(per_exec_ws('tvm/sanity')) { + // init_git() + // sh ( + // script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", + // label: 'Run lint', + // ) + // } + // } + // } + // } +} // Run make. First try to do an incremental make from a previous workspace in hope to // accelerate the compilation. If something is wrong, clean the workspace and then diff --git a/docker/build.sh b/docker/build.sh index 39fe7a024246..ed67b638c79b 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -24,6 +24,7 @@ # [--dockerfile ] [-it] # [--net=host] [--cache-from ] # [--name CONTAINER_NAME] [--context-path ] +# [--spec DOCKER_IMAGE_SPEC] # [] # # CONTAINER_TYPE: Type of the docker container used the run the build, @@ -36,6 +37,9 @@ # this optional value is not supplied (via the --dockerfile # flag), will use Dockerfile.CONTAINER_TYPE in default # +# DOCKER_IMAGE_SPEC: Override the default logic to determine the image name and +# tag +# # IMAGE_NAME: An image to be as a source for cached layers when building the # Docker image requested. # @@ -73,6 +77,11 @@ if [[ "$1" == "-it" ]]; then shift 1 fi +if [[ "$1" == "--spec" ]]; then + OVERRIDE_IMAGE_SPEC="$2" + shift 2 +fi + if [[ "$1" == "--net=host" ]]; then CI_DOCKER_EXTRA_PARAMS+=('--net=host') CI_DOCKER_BUILD_EXTRA_PARAMS+=("--network=host") @@ -162,6 +171,10 @@ DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]') # Compose the full image spec with "name:tag" e.g. "tvm.ci_cpu:v0.03" DOCKER_IMG_SPEC="${DOCKER_IMG_NAME}:${DOCKER_IMAGE_TAG}" +if [[ -n ${OVERRIDE_IMAGE_SPEC+x} ]]; then + DOCKER_IMG_SPEC="$OVERRIDE_IMAGE_SPEC" +fi + # Print arguments. echo "WORKSPACE: ${WORKSPACE}" echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}" diff --git a/tests/scripts/git_change_docker.sh b/tests/scripts/git_change_docker.sh new file mode 100755 index 000000000000..465492ddf85b --- /dev/null +++ b/tests/scripts/git_change_docker.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eux + +changed_files=$(git diff --no-commit-id --name-only -r origin/main) + +for file in $changed_files; do + echo "Checking $file" + if grep -q "docker/" <<< "$file"; then + exit 1 + fi +done + +# No docker changes +exit 0