diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000000..29e2373f30ff --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +Jenkinsfile linguist-generated=true + diff --git a/Jenkinsfile b/Jenkinsfile index 7b8c8f890db1..0b64f9306844 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-05-20T18:06:10.772162 +// Generated at 2022-05-20T13:24:01.371704 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -86,6 +86,20 @@ docker_build = 'docker/build.sh' max_time = 180 rebuild_docker_images = false +// skips builds from branch indexing; sourced from https://www.jvt.me/posts/2020/02/23/jenkins-multibranch-skip-branch-index/ +// execute this before anything else, including requesting any time on an agent +if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) { + print "INFO: Build skipped due to trigger being Branch Indexing" + currentBuild.result = 'ABORTED' // optional, gives a better hint to the user that it's been skipped, rather than the default which shows it's successful + return +} + +// Filenames for stashing between build and test steps +s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" + +// General note: Jenkins has limits on the size of a method (or top level code) +// that are pretty strict, so most usage of groovy methods in these templates +// are purely to satisfy the JVM def per_exec_ws(folder) { return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder } @@ -183,146 +197,52 @@ def should_skip_ci(pr_number) { return git_skip_ci_code == 0 } -// skips builds from branch indexing; sourced from https://www.jvt.me/posts/2020/02/23/jenkins-multibranch-skip-branch-index/ -// execute this before anything else, including requesting any time on an agent -if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) { - print "INFO: Build skipped due to trigger being Branch Indexing" - currentBuild.result = 'ABORTED' // optional, gives a better hint to the user that it's been skipped, rather than the default which shows it's successful - return -} - -cancel_previous_build() - -def lint() { -stage('Lint') { - parallel( - 'Lint 1 of 2': { +def prepare() { + stage('Prepare') { node('CPU-SMALL') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") { init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=0'], { - ci_arm = params.ci_arm_param ?: ci_arm - ci_cpu = params.ci_cpu_param ?: ci_cpu - ci_gpu = params.ci_gpu_param ?: ci_gpu - ci_hexagon = params.ci_hexagon_param ?: ci_hexagon - ci_i386 = params.ci_i386_param ?: ci_i386 - ci_lint = params.ci_lint_param ?: ci_lint - ci_qemu = params.ci_qemu_param ?: ci_qemu - ci_wasm = params.ci_wasm_param ?: ci_wasm - - sh (script: """ - echo "Docker images being used in this build:" - echo " ci_arm = ${ci_arm}" - echo " ci_cpu = ${ci_cpu}" - echo " ci_gpu = ${ci_gpu}" - echo " ci_hexagon = ${ci_hexagon}" - echo " ci_i386 = ${ci_i386}" - echo " ci_lint = ${ci_lint}" - echo " ci_qemu = ${ci_qemu}" - echo " ci_wasm = ${ci_wasm}" - """, label: 'Docker image names') - - is_docs_only_build = sh ( - returnStatus: true, - script: './tests/scripts/git_change_docs.sh', - label: 'Check for docs only changes', - ) - skip_ci = should_skip_ci(env.CHANGE_ID) - skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) - rebuild_docker_images = sh ( - returnStatus: true, - script: './tests/scripts/git_change_docker.sh', - label: 'Check for any docker changes', - ) - if (skip_ci) { - // Don't rebuild when skipping CI - rebuild_docker_images = false - } - if (rebuild_docker_images) { - // Exit before linting so we can use the newly created Docker images - // to run the lint - return - } - sh ( - script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", - label: 'Run lint', - ) - }) - } - } - } - }, - 'Lint 2 of 2': { - node('CPU-SMALL') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=1'], { - ci_arm = params.ci_arm_param ?: ci_arm - ci_cpu = params.ci_cpu_param ?: ci_cpu - ci_gpu = params.ci_gpu_param ?: ci_gpu - ci_hexagon = params.ci_hexagon_param ?: ci_hexagon - ci_i386 = params.ci_i386_param ?: ci_i386 - ci_lint = params.ci_lint_param ?: ci_lint - ci_qemu = params.ci_qemu_param ?: ci_qemu - ci_wasm = params.ci_wasm_param ?: ci_wasm - - sh (script: """ - echo "Docker images being used in this build:" - echo " ci_arm = ${ci_arm}" - echo " ci_cpu = ${ci_cpu}" - echo " ci_gpu = ${ci_gpu}" - echo " ci_hexagon = ${ci_hexagon}" - echo " ci_i386 = ${ci_i386}" - echo " ci_lint = ${ci_lint}" - echo " ci_qemu = ${ci_qemu}" - echo " ci_wasm = ${ci_wasm}" - """, label: 'Docker image names') - - is_docs_only_build = sh ( - returnStatus: true, - script: './tests/scripts/git_change_docs.sh', - label: 'Check for docs only changes', - ) - skip_ci = should_skip_ci(env.CHANGE_ID) - skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) - rebuild_docker_images = sh ( - returnStatus: true, - script: './tests/scripts/git_change_docker.sh', - label: 'Check for any docker changes', - ) - if (skip_ci) { - // Don't rebuild when skipping CI - rebuild_docker_images = false - } - if (rebuild_docker_images) { - // Exit before linting so we can use the newly created Docker images - // to run the lint - return - } - sh ( - script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", - label: 'Run lint', - ) - }) + ci_arm = params.ci_arm_param ?: ci_arm + ci_cpu = params.ci_cpu_param ?: ci_cpu + ci_gpu = params.ci_gpu_param ?: ci_gpu + ci_hexagon = params.ci_hexagon_param ?: ci_hexagon + ci_i386 = params.ci_i386_param ?: ci_i386 + ci_lint = params.ci_lint_param ?: ci_lint + ci_qemu = params.ci_qemu_param ?: ci_qemu + ci_wasm = params.ci_wasm_param ?: ci_wasm + + sh (script: """ + echo "Docker images being used in this build:" + echo " ci_arm = ${ci_arm}" + echo " ci_cpu = ${ci_cpu}" + echo " ci_gpu = ${ci_gpu}" + echo " ci_hexagon = ${ci_hexagon}" + echo " ci_i386 = ${ci_i386}" + echo " ci_lint = ${ci_lint}" + echo " ci_qemu = ${ci_qemu}" + echo " ci_wasm = ${ci_wasm}" + """, label: 'Docker image names') + + is_docs_only_build = sh ( + returnStatus: true, + script: './tests/scripts/git_change_docs.sh', + label: 'Check for docs only changes', + ) + skip_ci = should_skip_ci(env.CHANGE_ID) + skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) + rebuild_docker_images = sh ( + returnStatus: true, + script: './tests/scripts/git_change_docker.sh', + label: 'Check for any docker changes', + ) + if (skip_ci) { + // Don't rebuild when skipping CI + rebuild_docker_images = false } } } - }, - ) -} + } } - -// [note: method size] -// This has to be extracted into a method due to JVM limitations on the size of -// a method (so the code can't all be inlined) -lint() - def build_image(image_name) { hash = sh( returnStdout: true, @@ -378,7 +298,7 @@ def build_image(image_name) { ) } -if (rebuild_docker_images) { +def build_docker_images() { stage('Docker Image Build') { // TODO in a follow up PR: Find ecr tag and use in subsequent builds parallel 'ci-lint': { @@ -481,11 +401,46 @@ def make(docker_type, path, make_flag) { } } } - -// Filenames for stashing between build and test steps -s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" - - +def lint() { + stage('Lint') { + parallel( + 'Lint 1 of 2': { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'TVM_NUM_SHARDS=2', + 'TVM_SHARD_INDEX=0'], { + sh ( + script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", + label: 'Run lint', + ) + }) + } + } + } + }, + 'Lint 2 of 2': { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'TVM_NUM_SHARDS=2', + 'TVM_SHARD_INDEX=1'], { + sh ( + script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", + label: 'Run lint', + ) + }) + } + } + } + }, + ) + } +} def ci_setup(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_ci_setup.sh", @@ -529,7 +484,6 @@ def add_microtvm_permissions() { ) } - def build() { stage('Build') { environment { @@ -771,10 +725,6 @@ stage('Build') { ) } } - -// [note: method size] -build() - def test() { stage('Test') { environment { @@ -1845,10 +1795,6 @@ stage('Test') { ) } } - -// [note: method size] -test() - /* stage('Build packages') { parallel 'conda CPU': { @@ -1907,11 +1853,13 @@ def deploy_docs() { } } -stage('Deploy') { - if (env.BRANCH_NAME == 'main' && env.DOCS_DEPLOY_ENABLED == 'yes') { - node('CPU') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/deploy-docs") { - sh( + +def deploy() { + stage('Deploy') { + if (env.BRANCH_NAME == 'main' && env.DOCS_DEPLOY_ENABLED == 'yes') { + node('CPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/deploy-docs") { + sh( script: """ set -eux aws s3 cp --no-progress s3://${s3_prefix}/docs/docs.tgz docs.tgz @@ -1920,8 +1868,26 @@ stage('Deploy') { label: 'Download artifacts from S3', ) - deploy_docs() + deploy_docs() + } } } } } + + +cancel_previous_build() + +prepare() + +if (rebuild_docker_images) { + build_docker_images() +} + +lint() + +build() + +test() + +deploy() diff --git a/jenkins/Build.groovy.j2 b/jenkins/Build.groovy.j2 new file mode 100644 index 000000000000..c1715949175b --- /dev/null +++ b/jenkins/Build.groovy.j2 @@ -0,0 +1,186 @@ +def ci_setup(image) { + sh ( + script: "${docker_run} ${image} ./tests/scripts/task_ci_setup.sh", + label: 'Set up CI environment', + ) +} + +def python_unittest(image) { + sh ( + script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", + label: 'Run Python unit tests', + ) +} + +def fsim_test(image) { + sh ( + script: "${docker_run} ${image} ./tests/scripts/task_python_vta_fsim.sh", + label: 'Run VTA tests in FSIM', + ) +} + +def cmake_build(image, path, make_flag) { + sh ( + script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod", + label: 'Run cmake build', + ) +} + +def cpp_unittest(image) { + sh ( + script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh", + label: 'Build and run C++ tests', + ) +} + + +def add_microtvm_permissions() { + {% for folder in microtvm_template_projects %} + sh( + script: 'find {{ folder }} -type f | grep qemu-hack | xargs chmod +x', + label: 'Add execute permissions for microTVM files', + ) + {% endfor %} +} + +def build() { +stage('Build') { + environment { + SKIP_SLOW_TESTS = "${skip_slow_tests}" + } + parallel( + 'BUILD: GPU': { + if (!skip_ci) { + node('CPU-SMALL') { + ws({{ m.per_exec_ws('tvm/build-gpu') }}) { + init_git() + sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" + make("${ci_gpu} --no-gpu", 'build', '-j2') + {{ m.upload_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }} + + // compiler test + sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build2" + make("${ci_gpu} --no-gpu", 'build2', '-j2') + {{ m.upload_artifacts(tag='gpu2', filenames=tvm_multilib) }} + } + } + } + }, + 'BUILD: CPU': { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws({{ m.per_exec_ws('tvm/build-cpu') }}) { + init_git() + sh ( + script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", + label: 'Create CPU cmake config', + ) + make(ci_cpu, 'build', '-j2') + {{ m.upload_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} + timeout(time: max_time, unit: 'MINUTES') { + ci_setup(ci_cpu) + // sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh" + // TODO(@jroesch): need to resolve CI issue will turn back on in follow up patch + sh (script: "${docker_run} ${ci_cpu} ./tests/scripts/task_rust.sh", label: 'Rust build and test') + } + } + } + } else { + Utils.markStageSkippedForConditional('BUILD: CPU') + } + }, + 'BUILD: WASM': { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws({{ m.per_exec_ws('tvm/build-wasm') }}) { + init_git() + sh ( + script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", + label: 'Create WASM cmake config', + ) + make(ci_wasm, 'build', '-j2') + cpp_unittest(ci_wasm) + timeout(time: max_time, unit: 'MINUTES') { + ci_setup(ci_wasm) + sh ( + script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh", + label: 'Run WASM lint and tests', + ) + } + } + } + } else { + Utils.markStageSkippedForConditional('BUILD: WASM') + } + }, + 'BUILD: i386': { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws({{ m.per_exec_ws('tvm/build-i386') }}) { + init_git() + sh ( + script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build", + label: 'Create i386 cmake config', + ) + make(ci_i386, 'build', '-j2') + {{ m.upload_artifacts(tag='i386', filenames=tvm_multilib_tsim) }} + } + } + } else { + Utils.markStageSkippedForConditional('BUILD: i386') + } + }, + 'BUILD: arm': { + if (!skip_ci && is_docs_only_build != 1) { + node('ARM') { + ws({{ m.per_exec_ws('tvm/build-arm') }}) { + init_git() + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build", + label: 'Create ARM cmake config', + ) + make(ci_arm, 'build', '-j4') + {{ m.upload_artifacts(tag='arm', filenames=tvm_multilib) }} + } + } + } else { + Utils.markStageSkippedForConditional('BUILD: arm') + } + }, + 'BUILD: QEMU': { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws({{ m.per_exec_ws('tvm/build-qemu') }}) { + init_git() + sh ( + script: "${docker_run} ${ci_qemu} ./tests/scripts/task_config_build_qemu.sh build", + label: 'Create QEMU cmake config', + ) + make(ci_qemu, 'build', '-j2') + {{ m.upload_artifacts(tag='qemu', filenames=tvm_lib, folders=microtvm_template_projects) }} + } + } + } else { + Utils.markStageSkippedForConditional('BUILD: QEMU') + } + }, + 'BUILD: Hexagon': { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws({{ m.per_exec_ws('tvm/build-hexagon') }}) { + init_git() + sh ( + script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build", + label: 'Create Hexagon cmake config', + ) + make(ci_hexagon, 'build', '-j2') + {{ m.upload_artifacts(tag='hexagon', filenames=tvm_lib) }} + } + } + } else { + Utils.markStageSkippedForConditional('BUILD: Hexagon') + } + }, + ) +} +} diff --git a/jenkins/Deploy.groovy.j2 b/jenkins/Deploy.groovy.j2 new file mode 100644 index 000000000000..917f71ded1ff --- /dev/null +++ b/jenkins/Deploy.groovy.j2 @@ -0,0 +1,71 @@ +/* +stage('Build packages') { + parallel 'conda CPU': { + node('CPU') { + sh "${docker_run} tlcpack/conda-cpu ./conda/build_cpu.sh + } + }, + 'conda cuda': { + node('CPU') { + sh "${docker_run} tlcpack/conda-cuda90 ./conda/build_cuda.sh + sh "${docker_run} tlcpack/conda-cuda100 ./conda/build_cuda.sh + } + } +// Here we could upload the packages to anaconda for releases +// and/or the main branch +} +*/ + +def deploy_docs() { + // Note: This code must stay in the Jenkinsfile to ensure that it runs + // from a trusted context only + sh( + script: ''' + set -eux + rm -rf tvm-site + git clone -b $DOCS_DEPLOY_BRANCH --depth=1 https://github.com/apache/tvm-site + cd tvm-site + git status + git checkout -B $DOCS_DEPLOY_BRANCH + + rm -rf docs + mkdir -p docs + tar xf ../docs.tgz -C docs + COMMIT=$(cat docs/commit_hash) + git add . + git config user.name tvm-bot + git config user.email 95660001+tvm-bot@users.noreply.github.com + git commit -m"deploying docs (apache/tvm@$COMMIT)" + git status + ''', + label: 'Unpack docs and update tvm-site' + ) + + withCredentials([string( + credentialsId: 'docs-push-token', + variable: 'GITHUB_TOKEN', + )]) { + sh( + script: ''' + cd tvm-site + git remote add deploy https://$GITHUB_TOKEN:x-oauth-basic@github.com/apache/tvm-site.git + git push deploy $DOCS_DEPLOY_BRANCH + ''', + label: 'Upload docs to apache/tvm-site' + ) + } +} + + +def deploy() { + stage('Deploy') { + if (env.BRANCH_NAME == 'main' && env.DOCS_DEPLOY_ENABLED == 'yes') { + node('CPU') { + ws({{ m.per_exec_ws('tvm/deploy-docs') }}) { + {{ m.download_artifacts(tag='docs', filenames=["docs.tgz"]) }} + deploy_docs() + } + } + } + } +} diff --git a/jenkins/DockerBuild.groovy.j2 b/jenkins/DockerBuild.groovy.j2 new file mode 100644 index 000000000000..84bb8e3e376d --- /dev/null +++ b/jenkins/DockerBuild.groovy.j2 @@ -0,0 +1,158 @@ +def build_image(image_name) { + hash = sh( + returnStdout: true, + script: 'git log -1 --format=\'%h\'' + ).trim() + def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}" + sh( + script: "${docker_build} ${image_name} --spec ${full_name}", + label: 'Build docker image' + ) + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + // Use a credential so Jenkins knows to scrub the AWS account ID which is nice + // (but so we don't have to rely it being hardcoded in Jenkins) + withCredentials([string( + credentialsId: 'aws-account-id', + variable: '_ACCOUNT_ID_DO_NOT_USE', + )]) { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2']) { + sh( + script: ''' + set -x + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -x + docker tag ${full_name} \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name} + docker push \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name} + """, + label: 'Upload image to ECR' + ) + } + } + } finally { + sh( + script: 'rm -f ~/.docker/config.json', + label: 'Clean up login credentials' + ) + } + sh( + script: "docker rmi ${full_name}", + label: 'Remove docker image' + ) +} + +def build_docker_images() { + stage('Docker Image Build') { + // TODO in a follow up PR: Find ecr tag and use in subsequent builds + parallel 'ci-lint': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_lint') + } + } + }, 'ci-cpu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_cpu') + } + } + }, 'ci-gpu': { + node('GPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_gpu') + } + } + }, 'ci-qemu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_qemu') + } + } + }, 'ci-i386': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_i386') + } + } + }, 'ci-arm': { + node('ARM') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_arm') + } + } + }, 'ci-wasm': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_wasm') + } + } + }, 'ci-hexagon': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_hexagon') + } + } + } + } + // // TODO: Once we are able to use the built images, enable this step + // // If the docker images changed, we need to run the image build before the lint + // // can run since it requires a base docker image. Most of the time the images + // // aren't build though so it's faster to use the same node that checks for + // // docker changes to run the lint in the usual case. + // stage('Sanity Check (re-run)') { + // timeout(time: max_time, unit: 'MINUTES') { + // node('CPU') { + // ws({{ m.per_exec_ws('tvm/sanity') }}) { + // init_git() + // sh ( + // script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", + // label: 'Run lint', + // ) + // } + // } + // } + // } +} + +// Run make. First try to do an incremental make from a previous workspace in hope to +// accelerate the compilation. If something is wrong, clean the workspace and then +// build from scratch. +def make(docker_type, path, make_flag) { + timeout(time: max_time, unit: 'MINUTES') { + try { + cmake_build(docker_type, path, make_flag) + // always run cpp test when build + } catch (hudson.AbortException ae) { + // script exited due to user abort, directly throw instead of retry + if (ae.getMessage().contains('script returned exit code 143')) { + throw ae + } + echo 'Incremental compilation failed. Fall back to build from scratch' + sh ( + script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}", + label: 'Clear old cmake workspace', + ) + cmake_build(docker_type, path, make_flag) + } + } +} diff --git a/jenkins/Jenkinsfile.j2 b/jenkins/Jenkinsfile.j2 index b00ee0272626..a1127ec6a8d5 100644 --- a/jenkins/Jenkinsfile.j2 +++ b/jenkins/Jenkinsfile.j2 @@ -83,103 +83,6 @@ docker_build = 'docker/build.sh' max_time = 180 rebuild_docker_images = false -def per_exec_ws(folder) { - return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder -} - -// initialize source codes -def init_git() { - checkout scm - - // Clear out all Docker images that aren't going to be used - sh( - script: "docker image ls --all --format {% raw %}'{{.Repository}}:{{.Tag}} {{.ID}}'{% endraw %} | { grep -vE '{% for image in images %}{% raw %}${{% endraw %}{{ image.name }}{% raw %}}{% endraw %}{% if not loop.last %}|{% endif %}{% endfor %}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }", - label: 'Clean old Docker images', - ) - // Add more info about job node - sh ( - script: './tests/scripts/task_show_node_info.sh', - label: 'Show executor node info', - ) - - // Determine merge commit to use for all stages - sh ( - script: 'git fetch origin main', - label: 'Fetch upstream', - ) - if (upstream_revision == null) { - upstream_revision = sh( - script: 'git log -1 FETCH_HEAD --format=\'%H\'', - label: 'Determine upstream revision', - returnStdout: true, - ).trim() - } - sh ( - script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}", - label: 'Merge to origin/main' - ) - - retry(5) { - timeout(time: 2, unit: 'MINUTES') { - sh (script: 'git submodule update --init -f', label: 'Update git submodules') - } - } -} - -def should_skip_slow_tests(pr_number) { - withCredentials([string( - credentialsId: 'tvm-bot-jenkins-reader', - variable: 'GITHUB_TOKEN', - )]) { - // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests - result = sh ( - returnStatus: true, - script: "./tests/scripts/should_run_slow_tests.py --pr '${pr_number}'", - label: 'Check if CI should run slow tests', - ) - } - return result == 0 -} - -def cancel_previous_build() { - // cancel previous build if it is not on main. - if (env.BRANCH_NAME != 'main') { - def buildNumber = env.BUILD_NUMBER as int - // Milestone API allows us to cancel previous build - // with the same milestone number - if (buildNumber > 1) milestone(buildNumber - 1) - milestone(buildNumber) - } -} - -def should_skip_ci(pr_number) { - if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { - // never skip CI on build sourced from a branch - return false - } - glob_skip_ci_code = sh ( - returnStatus: true, - script: "./tests/scripts/git_skip_ci_globs.py", - label: 'Check if CI should be skipped due to changed files', - ) - if (glob_skip_ci_code == 0) { - return true - } - withCredentials([string( - credentialsId: 'tvm-bot-jenkins-reader', - variable: 'TOKEN', - )]) { - // Exit code of 1 means run full CI (or the script had an error, so run - // full CI just in case). Exit code of 0 means skip CI. - git_skip_ci_code = sh ( - returnStatus: true, - script: "./tests/scripts/git_skip_ci.py --pr '${pr_number}'", - label: 'Check if CI should be skipped', - ) - } - return git_skip_ci_code == 0 -} - // skips builds from branch indexing; sourced from https://www.jvt.me/posts/2020/02/23/jenkins-multibranch-skip-branch-index/ // execute this before anything else, including requesting any time on an agent if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) { @@ -188,217 +91,6 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) { return } -cancel_previous_build() - -def lint() { -stage('Lint') { - parallel( - {% call m.sharded_lint_step(name='Lint', num_shards=2, node='CPU-SMALL', ws='tvm/lint') %} - {% for image in images %} - {{ image.name }} = params.{{ image.name }}_param ?: {{ image.name }} - {% endfor %} - - sh (script: """ - echo "Docker images being used in this build:" - {% for image in images %} - echo " {{ image.name }} = ${ {{- image.name -}} }" - {% endfor %} - """, label: 'Docker image names') - - is_docs_only_build = sh ( - returnStatus: true, - script: './tests/scripts/git_change_docs.sh', - label: 'Check for docs only changes', - ) - skip_ci = should_skip_ci(env.CHANGE_ID) - skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) - rebuild_docker_images = sh ( - returnStatus: true, - script: './tests/scripts/git_change_docker.sh', - label: 'Check for any docker changes', - ) - if (skip_ci) { - // Don't rebuild when skipping CI - rebuild_docker_images = false - } - if (rebuild_docker_images) { - // Exit before linting so we can use the newly created Docker images - // to run the lint - return - } - sh ( - script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", - label: 'Run lint', - ) - {% endcall %} - ) -} -} - -// [note: method size] -// This has to be extracted into a method due to JVM limitations on the size of -// a method (so the code can't all be inlined) -lint() - -def build_image(image_name) { - hash = sh( - returnStdout: true, - script: 'git log -1 --format=\'%h\'' - ).trim() - def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}" - sh( - script: "${docker_build} ${image_name} --spec ${full_name}", - label: 'Build docker image' - ) - aws_account_id = sh( - returnStdout: true, - script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', - label: 'Get AWS ID' - ).trim() - - try { - // Use a credential so Jenkins knows to scrub the AWS account ID which is nice - // (but so we don't have to rely it being hardcoded in Jenkins) - withCredentials([string( - credentialsId: 'aws-account-id', - variable: '_ACCOUNT_ID_DO_NOT_USE', - )]) { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2']) { - sh( - script: ''' - set -x - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com - ''', - label: 'Log in to ECR' - ) - sh( - script: """ - set -x - docker tag ${full_name} \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name} - docker push \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name} - """, - label: 'Upload image to ECR' - ) - } - } - } finally { - sh( - script: 'rm -f ~/.docker/config.json', - label: 'Clean up login credentials' - ) - } - sh( - script: "docker rmi ${full_name}", - label: 'Remove docker image' - ) -} - -if (rebuild_docker_images) { - stage('Docker Image Build') { - // TODO in a follow up PR: Find ecr tag and use in subsequent builds - parallel 'ci-lint': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_lint') - } - } - }, 'ci-cpu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_cpu') - } - } - }, 'ci-gpu': { - node('GPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_gpu') - } - } - }, 'ci-qemu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_qemu') - } - } - }, 'ci-i386': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_i386') - } - } - }, 'ci-arm': { - node('ARM') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_arm') - } - } - }, 'ci-wasm': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_wasm') - } - } - }, 'ci-hexagon': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_hexagon') - } - } - } - } - // // TODO: Once we are able to use the built images, enable this step - // // If the docker images changed, we need to run the image build before the lint - // // can run since it requires a base docker image. Most of the time the images - // // aren't build though so it's faster to use the same node that checks for - // // docker changes to run the lint in the usual case. - // stage('Sanity Check (re-run)') { - // timeout(time: max_time, unit: 'MINUTES') { - // node('CPU') { - // ws({{ m.per_exec_ws('tvm/sanity') }}) { - // init_git() - // sh ( - // script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", - // label: 'Run lint', - // ) - // } - // } - // } - // } -} - -// Run make. First try to do an incremental make from a previous workspace in hope to -// accelerate the compilation. If something is wrong, clean the workspace and then -// build from scratch. -def make(docker_type, path, make_flag) { - timeout(time: max_time, unit: 'MINUTES') { - try { - cmake_build(docker_type, path, make_flag) - // always run cpp test when build - } catch (hudson.AbortException ae) { - // script exited due to user abort, directly throw instead of retry - if (ae.getMessage().contains('script returned exit code 143')) { - throw ae - } - echo 'Incremental compilation failed. Fall back to build from scratch' - sh ( - script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}", - label: 'Clear old cmake workspace', - ) - cmake_build(docker_type, path, make_flag) - } - } -} - // Filenames for stashing between build and test steps {% set tvm_runtime = ['build/libtvm_runtime.so', 'build/config.cmake'] %} {% set tvm_lib = ['build/libtvm.so'] + tvm_runtime %} @@ -407,503 +99,29 @@ def make(docker_type, path, make_flag) { {% set microtvm_template_projects = ['build/microtvm_template_projects',] %} s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" +// General note: Jenkins has limits on the size of a method (or top level code) +// that are pretty strict, so most usage of groovy methods in these templates +// are purely to satisfy the JVM +{% include "jenkins/Prepare.groovy.j2" %} +{% include "jenkins/DockerBuild.groovy.j2" %} +{% include "jenkins/Lint.groovy.j2" %} +{% include "jenkins/Build.groovy.j2" %} +{% include "jenkins/Test.groovy.j2" %} +{% include "jenkins/Deploy.groovy.j2" %} -def ci_setup(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_ci_setup.sh", - label: 'Set up CI environment', - ) -} - -def python_unittest(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", - label: 'Run Python unit tests', - ) -} - -def fsim_test(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_python_vta_fsim.sh", - label: 'Run VTA tests in FSIM', - ) -} - -def cmake_build(image, path, make_flag) { - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod", - label: 'Run cmake build', - ) -} -def cpp_unittest(image) { - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh", - label: 'Build and run C++ tests', - ) -} +cancel_previous_build() +prepare() -def add_microtvm_permissions() { - {% for folder in microtvm_template_projects %} - sh( - script: 'find {{ folder }} -type f | grep qemu-hack | xargs chmod +x', - label: 'Add execute permissions for microTVM files', - ) - {% endfor %} +if (rebuild_docker_images) { + build_docker_images() } +lint() -def build() { -stage('Build') { - environment { - SKIP_SLOW_TESTS = "${skip_slow_tests}" - } - parallel( - 'BUILD: GPU': { - if (!skip_ci) { - node('CPU-SMALL') { - ws({{ m.per_exec_ws('tvm/build-gpu') }}) { - init_git() - sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" - make("${ci_gpu} --no-gpu", 'build', '-j2') - {{ m.upload_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }} - - // compiler test - sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build2" - make("${ci_gpu} --no-gpu", 'build2', '-j2') - {{ m.upload_artifacts(tag='gpu2', filenames=tvm_multilib) }} - } - } - } - }, - 'BUILD: CPU': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws({{ m.per_exec_ws('tvm/build-cpu') }}) { - init_git() - sh ( - script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", - label: 'Create CPU cmake config', - ) - make(ci_cpu, 'build', '-j2') - {{ m.upload_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} - timeout(time: max_time, unit: 'MINUTES') { - ci_setup(ci_cpu) - // sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh" - // TODO(@jroesch): need to resolve CI issue will turn back on in follow up patch - sh (script: "${docker_run} ${ci_cpu} ./tests/scripts/task_rust.sh", label: 'Rust build and test') - } - } - } - } else { - Utils.markStageSkippedForConditional('BUILD: CPU') - } - }, - 'BUILD: WASM': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws({{ m.per_exec_ws('tvm/build-wasm') }}) { - init_git() - sh ( - script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", - label: 'Create WASM cmake config', - ) - make(ci_wasm, 'build', '-j2') - cpp_unittest(ci_wasm) - timeout(time: max_time, unit: 'MINUTES') { - ci_setup(ci_wasm) - sh ( - script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh", - label: 'Run WASM lint and tests', - ) - } - } - } - } else { - Utils.markStageSkippedForConditional('BUILD: WASM') - } - }, - 'BUILD: i386': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws({{ m.per_exec_ws('tvm/build-i386') }}) { - init_git() - sh ( - script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build", - label: 'Create i386 cmake config', - ) - make(ci_i386, 'build', '-j2') - {{ m.upload_artifacts(tag='i386', filenames=tvm_multilib_tsim) }} - } - } - } else { - Utils.markStageSkippedForConditional('BUILD: i386') - } - }, - 'BUILD: arm': { - if (!skip_ci && is_docs_only_build != 1) { - node('ARM') { - ws({{ m.per_exec_ws('tvm/build-arm') }}) { - init_git() - sh ( - script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build", - label: 'Create ARM cmake config', - ) - make(ci_arm, 'build', '-j4') - {{ m.upload_artifacts(tag='arm', filenames=tvm_multilib) }} - } - } - } else { - Utils.markStageSkippedForConditional('BUILD: arm') - } - }, - 'BUILD: QEMU': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws({{ m.per_exec_ws('tvm/build-qemu') }}) { - init_git() - sh ( - script: "${docker_run} ${ci_qemu} ./tests/scripts/task_config_build_qemu.sh build", - label: 'Create QEMU cmake config', - ) - make(ci_qemu, 'build', '-j2') - {{ m.upload_artifacts(tag='qemu', filenames=tvm_lib, folders=microtvm_template_projects) }} - } - } - } else { - Utils.markStageSkippedForConditional('BUILD: QEMU') - } - }, - 'BUILD: Hexagon': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws({{ m.per_exec_ws('tvm/build-hexagon') }}) { - init_git() - sh ( - script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build", - label: 'Create Hexagon cmake config', - ) - make(ci_hexagon, 'build', '-j2') - {{ m.upload_artifacts(tag='hexagon', filenames=tvm_lib) }} - } - } - } else { - Utils.markStageSkippedForConditional('BUILD: Hexagon') - } - }, - ) -} -} - -// [note: method size] build() -def test() { -stage('Test') { - environment { - SKIP_SLOW_TESTS = "${skip_slow_tests}" - } - parallel( - {% call(shard_index, num_shards) m.sharded_test_step( - name="unittest: GPU", - num_shards=2, - node="GPU", - ws="tvm/ut-python-gpu", - platform="gpu", - ) %} - {% if shard_index == 1 %} - {{ m.download_artifacts(tag='gpu2', filenames=tvm_multilib) }} - cpp_unittest(ci_gpu) - - {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} - ci_setup(ci_gpu) - cpp_unittest(ci_gpu) - {% else %} - {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} - ci_setup(ci_gpu) - {% endif %} - {% if shard_index == 2 or num_shards < 2 %} - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh", - label: 'Run Java unit tests', - ) - {% endif %} - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", - label: 'Run Python GPU unit tests', - ) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh", - label: 'Run Python GPU integration tests', - ) - {% endcall %} - {% call(shard_index, num_shards) m.sharded_test_step( - name="integration: CPU", - node="CPU", - num_shards=2, - ws="tvm/integration-python-cpu", - platform="cpu", - ) %} - {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} - ci_setup(ci_cpu) - sh ( - script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", - label: 'Run CPU integration tests', - ) - {% endcall %} - {% call m.test_step( - name="unittest: CPU", - node="CPU-SMALL", - ws="tvm/ut-python-cpu", - platform="cpu", - ) %} - {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} - ci_setup(ci_cpu) - cpp_unittest(ci_cpu) - python_unittest(ci_cpu) - fsim_test(ci_cpu) - sh ( - script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_vta_tsim.sh", - label: 'Run VTA tests in TSIM', - ) - {% endcall %} - {% call(shard_index, num_shards) m.sharded_test_step( - name="python: i386", - node="CPU-SMALL", - num_shards=3, - ws="tvm/integration-python-i386", - platform="i386", - ) %} - {{ m.download_artifacts(tag='i386', filenames=tvm_multilib) }} - ci_setup(ci_i386) - {% if shard_index == 1 %} - cpp_unittest(ci_i386) - {% endif %} - python_unittest(ci_i386) - sh ( - script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh", - label: 'Run i386 integration tests', - ) - fsim_test(ci_i386) - {% endcall %} - {% call(shard_index, num_shards) m.sharded_test_step( - name="test: Hexagon", - node="CPU-SMALL", - ws="tvm/test-hexagon", - platform="hexagon", - num_shards=4, - ) %} - {{ m.download_artifacts(tag='hexagon', filenames=tvm_lib) }} - ci_setup(ci_hexagon) - {% if shard_index == 1 %} - cpp_unittest(ci_hexagon) - {% endif %} - sh ( - script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh", - label: 'Build Hexagon API', - ) - sh ( - script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh", - label: 'Run Hexagon tests', - ) - {% endcall %} - {% call m.test_step( - name="test: QEMU", - node="CPU-SMALL", - ws="tvm/test-qemu", - platform="qemu", - ) %} - {{ m.download_artifacts(tag='qemu', filenames=tvm_lib, folders=microtvm_template_projects) }} - add_microtvm_permissions() - ci_setup(ci_qemu) - cpp_unittest(ci_qemu) - sh ( - script: "${docker_run} ${ci_qemu} ./tests/scripts/task_python_microtvm.sh", - label: 'Run microTVM tests', - ) - sh ( - script: "${docker_run} ${ci_qemu} ./tests/scripts/task_demo_microtvm.sh", - label: 'Run microTVM demos', - ) - {% endcall %} - {% call m.test_step( - name="topi: aarch64", - node="ARM", - ws="tvm/ut-python-arm", - platform="arm", -) %} - {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} - ci_setup(ci_arm) - cpp_unittest(ci_arm) - sh ( - script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_arm_compute_library.sh", - label: 'Run test_arm_compute_lib test', - ) - sh ( - script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_topi.sh", - label: 'Run TOPI tests', - ) - {% endcall %} - {% call(shard_index, num_shards) m.sharded_test_step( - name="integration: aarch64", - num_shards=2, - node="ARM", ws="tvm/ut-python-arm", - platform="arm", - ) %} - {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} - ci_setup(ci_arm) - python_unittest(ci_arm) - sh ( - script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh", - label: 'Run CPU integration tests', - ) - {% endcall %} - {% call(shard_index, num_shards) m.sharded_test_step( - name="topi: GPU", - node="GPU", - num_shards=2, - ws="tvm/topi-python-gpu", - platform="gpu", - ) %} - {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} - ci_setup(ci_gpu) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", - label: 'Run TOPI tests', - ) - {% endcall %} - {% call(shard_index, num_shards) m.sharded_test_step( - name="frontend: GPU", node="GPU", - num_shards=3, - ws="tvm/frontend-python-gpu", - platform="gpu", - ) %} - {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} - ci_setup(ci_gpu) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", - label: 'Run Python frontend tests', - ) - {% endcall %} - {% call m.test_step( - name="frontend: CPU", - node="CPU", - ws="tvm/frontend-python-cpu", - platform="cpu", -) %} - {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib) }} - ci_setup(ci_cpu) - sh ( - script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_frontend_cpu.sh", - label: 'Run Python frontend tests', - ) - {% endcall %} - {% call m.test_step( - name="frontend: aarch64", - node="ARM", - ws="tvm/frontend-python-arm", - platform="arm", -) %} - {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} - ci_setup(ci_arm) - sh ( - script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh", - label: 'Run Python frontend tests', - ) - {% endcall %} - 'docs: GPU': { - if (!skip_ci) { - node('GPU') { - ws({{ m.per_exec_ws('tvm/docs-python-gpu') }}) { - init_git() - {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }} - add_microtvm_permissions() - timeout(time: 180, unit: 'MINUTES') { - ci_setup(ci_gpu) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh", - label: 'Build docs', - ) - } - {{ m.upload_artifacts(tag='docs', filenames=["docs.tgz"]) }} - archiveArtifacts(artifacts: 'docs.tgz', fingerprint: true) - } - } - } - }, - ) -} -} - -// [note: method size] test() -/* -stage('Build packages') { - parallel 'conda CPU': { - node('CPU') { - sh "${docker_run} tlcpack/conda-cpu ./conda/build_cpu.sh - } - }, - 'conda cuda': { - node('CPU') { - sh "${docker_run} tlcpack/conda-cuda90 ./conda/build_cuda.sh - sh "${docker_run} tlcpack/conda-cuda100 ./conda/build_cuda.sh - } - } -// Here we could upload the packages to anaconda for releases -// and/or the main branch -} -*/ - -def deploy_docs() { - // Note: This code must stay in the Jenkinsfile to ensure that it runs - // from a trusted context only - sh( - script: ''' - set -eux - rm -rf tvm-site - git clone -b $DOCS_DEPLOY_BRANCH --depth=1 https://github.com/apache/tvm-site - cd tvm-site - git status - git checkout -B $DOCS_DEPLOY_BRANCH - - rm -rf docs - mkdir -p docs - tar xf ../docs.tgz -C docs - COMMIT=$(cat docs/commit_hash) - git add . - git config user.name tvm-bot - git config user.email 95660001+tvm-bot@users.noreply.github.com - git commit -m"deploying docs (apache/tvm@$COMMIT)" - git status - ''', - label: 'Unpack docs and update tvm-site' - ) - - withCredentials([string( - credentialsId: 'docs-push-token', - variable: 'GITHUB_TOKEN', - )]) { - sh( - script: ''' - cd tvm-site - git remote add deploy https://$GITHUB_TOKEN:x-oauth-basic@github.com/apache/tvm-site.git - git push deploy $DOCS_DEPLOY_BRANCH - ''', - label: 'Upload docs to apache/tvm-site' - ) - } -} - -stage('Deploy') { - if (env.BRANCH_NAME == 'main' && env.DOCS_DEPLOY_ENABLED == 'yes') { - node('CPU') { - ws({{ m.per_exec_ws('tvm/deploy-docs') }}) { - {{ m.download_artifacts(tag='docs', filenames=["docs.tgz"]) }} - deploy_docs() - } - } - } -} +deploy() diff --git a/jenkins/Lint.groovy.j2 b/jenkins/Lint.groovy.j2 new file mode 100644 index 000000000000..61c13cd407d0 --- /dev/null +++ b/jenkins/Lint.groovy.j2 @@ -0,0 +1,18 @@ +def lint() { + stage('Lint') { + parallel( + {% call m.sharded_lint_step( + name='Lint', + num_shards=2, + node='CPU-SMALL', + ws='tvm/lint', + ) + %} + sh ( + script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", + label: 'Run lint', + ) + {% endcall %} + ) + } +} diff --git a/jenkins/Prepare.groovy.j2 b/jenkins/Prepare.groovy.j2 new file mode 100644 index 000000000000..d7bf5e706b0b --- /dev/null +++ b/jenkins/Prepare.groovy.j2 @@ -0,0 +1,133 @@ +def per_exec_ws(folder) { + return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder +} + +// initialize source codes +def init_git() { + checkout scm + + // Clear out all Docker images that aren't going to be used + sh( + script: "docker image ls --all --format {% raw %}'{{.Repository}}:{{.Tag}} {{.ID}}'{% endraw %} | { grep -vE '{% for image in images %}{% raw %}${{% endraw %}{{ image.name }}{% raw %}}{% endraw %}{% if not loop.last %}|{% endif %}{% endfor %}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }", + label: 'Clean old Docker images', + ) + // Add more info about job node + sh ( + script: './tests/scripts/task_show_node_info.sh', + label: 'Show executor node info', + ) + + // Determine merge commit to use for all stages + sh ( + script: 'git fetch origin main', + label: 'Fetch upstream', + ) + if (upstream_revision == null) { + upstream_revision = sh( + script: 'git log -1 FETCH_HEAD --format=\'%H\'', + label: 'Determine upstream revision', + returnStdout: true, + ).trim() + } + sh ( + script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}", + label: 'Merge to origin/main' + ) + + retry(5) { + timeout(time: 2, unit: 'MINUTES') { + sh (script: 'git submodule update --init -f', label: 'Update git submodules') + } + } +} + +def should_skip_slow_tests(pr_number) { + withCredentials([string( + credentialsId: 'tvm-bot-jenkins-reader', + variable: 'GITHUB_TOKEN', + )]) { + // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests + result = sh ( + returnStatus: true, + script: "./tests/scripts/should_run_slow_tests.py --pr '${pr_number}'", + label: 'Check if CI should run slow tests', + ) + } + return result == 0 +} + +def cancel_previous_build() { + // cancel previous build if it is not on main. + if (env.BRANCH_NAME != 'main') { + def buildNumber = env.BUILD_NUMBER as int + // Milestone API allows us to cancel previous build + // with the same milestone number + if (buildNumber > 1) milestone(buildNumber - 1) + milestone(buildNumber) + } +} + +def should_skip_ci(pr_number) { + if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { + // never skip CI on build sourced from a branch + return false + } + glob_skip_ci_code = sh ( + returnStatus: true, + script: "./tests/scripts/git_skip_ci_globs.py", + label: 'Check if CI should be skipped due to changed files', + ) + if (glob_skip_ci_code == 0) { + return true + } + withCredentials([string( + credentialsId: 'tvm-bot-jenkins-reader', + variable: 'TOKEN', + )]) { + // Exit code of 1 means run full CI (or the script had an error, so run + // full CI just in case). Exit code of 0 means skip CI. + git_skip_ci_code = sh ( + returnStatus: true, + script: "./tests/scripts/git_skip_ci.py --pr '${pr_number}'", + label: 'Check if CI should be skipped', + ) + } + return git_skip_ci_code == 0 +} + +def prepare() { + stage('Prepare') { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") { + init_git() + {% for image in images %} + {{ image.name }} = params.{{ image.name }}_param ?: {{ image.name }} + {% endfor %} + + sh (script: """ + echo "Docker images being used in this build:" + {% for image in images %} + echo " {{ image.name }} = ${ {{- image.name -}} }" + {% endfor %} + """, label: 'Docker image names') + + is_docs_only_build = sh ( + returnStatus: true, + script: './tests/scripts/git_change_docs.sh', + label: 'Check for docs only changes', + ) + skip_ci = should_skip_ci(env.CHANGE_ID) + skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) + rebuild_docker_images = sh ( + returnStatus: true, + script: './tests/scripts/git_change_docker.sh', + label: 'Check for any docker changes', + ) + if (skip_ci) { + // Don't rebuild when skipping CI + rebuild_docker_images = false + } + } + } + } +} diff --git a/jenkins/README.md b/jenkins/README.md new file mode 100644 index 000000000000..454664b40c64 --- /dev/null +++ b/jenkins/README.md @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + +# Jenkins CI + +The template files in this directory are used to generate the [`Jenkinsfile`](../Jenkinsfile) used by Jenkins to run CI jobs for each commit to PRs and branches. + +To regenerate the `Jenkinsfile`, run + +```bash +pip install -r jenkins/requirements.txt +python jenkins/generate.py +``` + diff --git a/jenkins/Test.groovy.j2 b/jenkins/Test.groovy.j2 new file mode 100644 index 000000000000..b287c2a3156e --- /dev/null +++ b/jenkins/Test.groovy.j2 @@ -0,0 +1,236 @@ +def test() { +stage('Test') { + environment { + SKIP_SLOW_TESTS = "${skip_slow_tests}" + } + parallel( + {% call(shard_index, num_shards) m.sharded_test_step( + name="unittest: GPU", + num_shards=2, + node="GPU", + ws="tvm/ut-python-gpu", + platform="gpu", + ) %} + {% if shard_index == 1 %} + {{ m.download_artifacts(tag='gpu2', filenames=tvm_multilib) }} + cpp_unittest(ci_gpu) + + {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} + ci_setup(ci_gpu) + cpp_unittest(ci_gpu) + {% else %} + {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} + ci_setup(ci_gpu) + {% endif %} + {% if shard_index == 2 or num_shards < 2 %} + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh", + label: 'Run Java unit tests', + ) + {% endif %} + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", + label: 'Run Python GPU unit tests', + ) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh", + label: 'Run Python GPU integration tests', + ) + {% endcall %} + {% call(shard_index, num_shards) m.sharded_test_step( + name="integration: CPU", + node="CPU", + num_shards=2, + ws="tvm/integration-python-cpu", + platform="cpu", + ) %} + {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} + ci_setup(ci_cpu) + sh ( + script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", + label: 'Run CPU integration tests', + ) + {% endcall %} + {% call m.test_step( + name="unittest: CPU", + node="CPU-SMALL", + ws="tvm/ut-python-cpu", + platform="cpu", + ) %} + {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} + ci_setup(ci_cpu) + cpp_unittest(ci_cpu) + python_unittest(ci_cpu) + fsim_test(ci_cpu) + sh ( + script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_vta_tsim.sh", + label: 'Run VTA tests in TSIM', + ) + {% endcall %} + {% call(shard_index, num_shards) m.sharded_test_step( + name="python: i386", + node="CPU-SMALL", + num_shards=3, + ws="tvm/integration-python-i386", + platform="i386", + ) %} + {{ m.download_artifacts(tag='i386', filenames=tvm_multilib) }} + ci_setup(ci_i386) + {% if shard_index == 1 %} + cpp_unittest(ci_i386) + {% endif %} + python_unittest(ci_i386) + sh ( + script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh", + label: 'Run i386 integration tests', + ) + fsim_test(ci_i386) + {% endcall %} + {% call(shard_index, num_shards) m.sharded_test_step( + name="test: Hexagon", + node="CPU-SMALL", + ws="tvm/test-hexagon", + platform="hexagon", + num_shards=4, + ) %} + {{ m.download_artifacts(tag='hexagon', filenames=tvm_lib) }} + ci_setup(ci_hexagon) + {% if shard_index == 1 %} + cpp_unittest(ci_hexagon) + {% endif %} + sh ( + script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh", + label: 'Build Hexagon API', + ) + sh ( + script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh", + label: 'Run Hexagon tests', + ) + {% endcall %} + {% call m.test_step( + name="test: QEMU", + node="CPU-SMALL", + ws="tvm/test-qemu", + platform="qemu", + ) %} + {{ m.download_artifacts(tag='qemu', filenames=tvm_lib, folders=microtvm_template_projects) }} + add_microtvm_permissions() + ci_setup(ci_qemu) + cpp_unittest(ci_qemu) + sh ( + script: "${docker_run} ${ci_qemu} ./tests/scripts/task_python_microtvm.sh", + label: 'Run microTVM tests', + ) + sh ( + script: "${docker_run} ${ci_qemu} ./tests/scripts/task_demo_microtvm.sh", + label: 'Run microTVM demos', + ) + {% endcall %} + {% call m.test_step( + name="topi: aarch64", + node="ARM", + ws="tvm/ut-python-arm", + platform="arm", +) %} + {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} + ci_setup(ci_arm) + cpp_unittest(ci_arm) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_arm_compute_library.sh", + label: 'Run test_arm_compute_lib test', + ) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_topi.sh", + label: 'Run TOPI tests', + ) + {% endcall %} + {% call(shard_index, num_shards) m.sharded_test_step( + name="integration: aarch64", + num_shards=2, + node="ARM", ws="tvm/ut-python-arm", + platform="arm", + ) %} + {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} + ci_setup(ci_arm) + python_unittest(ci_arm) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh", + label: 'Run CPU integration tests', + ) + {% endcall %} + {% call(shard_index, num_shards) m.sharded_test_step( + name="topi: GPU", + node="GPU", + num_shards=2, + ws="tvm/topi-python-gpu", + platform="gpu", + ) %} + {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", + label: 'Run TOPI tests', + ) + {% endcall %} + {% call(shard_index, num_shards) m.sharded_test_step( + name="frontend: GPU", node="GPU", + num_shards=3, + ws="tvm/frontend-python-gpu", + platform="gpu", + ) %} + {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", + label: 'Run Python frontend tests', + ) + {% endcall %} + {% call m.test_step( + name="frontend: CPU", + node="CPU", + ws="tvm/frontend-python-cpu", + platform="cpu", +) %} + {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib) }} + ci_setup(ci_cpu) + sh ( + script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_frontend_cpu.sh", + label: 'Run Python frontend tests', + ) + {% endcall %} + {% call m.test_step( + name="frontend: aarch64", + node="ARM", + ws="tvm/frontend-python-arm", + platform="arm", +) %} + {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} + ci_setup(ci_arm) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh", + label: 'Run Python frontend tests', + ) + {% endcall %} + 'docs: GPU': { + if (!skip_ci) { + node('GPU') { + ws({{ m.per_exec_ws('tvm/docs-python-gpu') }}) { + init_git() + {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }} + add_microtvm_permissions() + timeout(time: 180, unit: 'MINUTES') { + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh", + label: 'Build docs', + ) + } + {{ m.upload_artifacts(tag='docs', filenames=["docs.tgz"]) }} + archiveArtifacts(artifacts: 'docs.tgz', fingerprint: true) + } + } + } + }, + ) +} +} diff --git a/tests/lint/rat-excludes b/tests/lint/rat-excludes index 3dff79c565ce..1cdb78e31913 100644 --- a/tests/lint/rat-excludes +++ b/tests/lint/rat-excludes @@ -51,3 +51,11 @@ MANIFEST .bash_history rat-excludes Cargo.lock + +# Included template files +Build.groovy.j2 +Deploy.groovy.j2 +DockerBuild.groovy.j2 +Lint.groovy.j2 +Prepare.groovy.j2 +Test.groovy.j2