Skip to content

Commit 1599755

Browse files
driazatidriazati
committed
[ci] Split hexagon into 2 steps
This shards up the hexagon build since after #11016 it's the longest test step. This also drops the max runtime per stage down to 2 hours from 4 hours to make these kind of increases more obvious in the future.
1 parent 6846484 commit 1599755

File tree

3 files changed

+149
-23
lines changed

3 files changed

+149
-23
lines changed

‎Jenkinsfile‎

Lines changed: 123 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
// 'python3 jenkins/generate.py'
4646
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
4747
// always rebased on main before merging:
48-
// Generated at 2022-04-22T12:59:15.071304
48+
// Generated at 2022-04-26T15:08:12.905694
4949

5050
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
5151
// NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. -->
@@ -92,7 +92,7 @@ upstream_revision = null
9292
docker_run = 'docker/bash.sh --env CI --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM'
9393
docker_build = 'docker/build.sh'
9494
// timeout in minutes
95-
max_time = 240
95+
max_time = 120
9696
rebuild_docker_images = false
9797

9898
def per_exec_ws(folder) {
@@ -196,7 +196,7 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) {
196196

197197
cancel_previous_build()
198198

199-
199+
def lint() {
200200
stage('Lint') {
201201
node('CPU') {
202202
timeout(time: max_time, unit: 'MINUTES') {
@@ -252,6 +252,11 @@ stage('Lint') {
252252
}
253253
}
254254
}
255+
}
256+
257+
// This has to be extracted into a method due to JVM limitations on the size of
258+
// a method (so the code can't all be inlined)
259+
lint()
255260

256261
def build_image(image_name) {
257262
hash = sh(
@@ -465,6 +470,7 @@ def cpp_unittest(image) {
465470
)
466471
}
467472

473+
def build() {
468474
stage('Build') {
469475
environment {
470476
SKIP_SLOW_TESTS = "${skip_slow_tests}"
@@ -605,7 +611,10 @@ stage('Build') {
605611
}
606612
}
607613
}
614+
}
615+
build()
608616

617+
def test() {
609618
stage('Test') {
610619
environment {
611620
SKIP_SLOW_TESTS = "${skip_slow_tests}"
@@ -837,17 +846,20 @@ stage('Test') {
837846
Utils.markStageSkippedForConditional('python: i386 2 of 2')
838847
}
839848
},
840-
'test: Hexagon': {
849+
'test: Hexagon 1 of 4': {
841850
if (!skip_ci && is_docs_only_build != 1) {
842851
node('CPU') {
843852
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
844-
timeout(time: max_time, unit: 'MINUTES') {
845-
try {
846-
init_git()
847-
withEnv(['PLATFORM=hexagon'], {
853+
try {
854+
init_git()
855+
timeout(time: max_time, unit: 'MINUTES') {
856+
withEnv([
857+
'PLATFORM=hexagon',
858+
'TVM_NUM_SHARDS=4',
859+
'TVM_SHARD_INDEX=0'], {
848860
unpack_lib('hexagon', tvm_lib)
849861
ci_setup(ci_hexagon)
850-
cpp_unittest(ci_hexagon)
862+
cpp_unittest(ci_hexagon)
851863
sh (
852864
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh",
853865
label: 'Build Hexagon API',
@@ -857,14 +869,110 @@ stage('Test') {
857869
label: 'Run Hexagon tests',
858870
)
859871
})
860-
} finally {
861-
junit 'build/pytest-results/*.xml'
862872
}
873+
} finally {
874+
junit 'build/pytest-results/*.xml'
863875
}
864876
}
865877
}
866878
} else {
867-
Utils.markStageSkippedForConditional('test: Hexagon')
879+
Utils.markStageSkippedForConditional('test: Hexagon 1 of 4')
880+
}
881+
},
882+
'test: Hexagon 2 of 4': {
883+
if (!skip_ci && is_docs_only_build != 1) {
884+
node('CPU') {
885+
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
886+
try {
887+
init_git()
888+
timeout(time: max_time, unit: 'MINUTES') {
889+
withEnv([
890+
'PLATFORM=hexagon',
891+
'TVM_NUM_SHARDS=4',
892+
'TVM_SHARD_INDEX=1'], {
893+
unpack_lib('hexagon', tvm_lib)
894+
ci_setup(ci_hexagon)
895+
sh (
896+
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh",
897+
label: 'Build Hexagon API',
898+
)
899+
sh (
900+
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
901+
label: 'Run Hexagon tests',
902+
)
903+
})
904+
}
905+
} finally {
906+
junit 'build/pytest-results/*.xml'
907+
}
908+
}
909+
}
910+
} else {
911+
Utils.markStageSkippedForConditional('test: Hexagon 2 of 4')
912+
}
913+
},
914+
'test: Hexagon 3 of 4': {
915+
if (!skip_ci && is_docs_only_build != 1) {
916+
node('CPU') {
917+
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
918+
try {
919+
init_git()
920+
timeout(time: max_time, unit: 'MINUTES') {
921+
withEnv([
922+
'PLATFORM=hexagon',
923+
'TVM_NUM_SHARDS=4',
924+
'TVM_SHARD_INDEX=2'], {
925+
unpack_lib('hexagon', tvm_lib)
926+
ci_setup(ci_hexagon)
927+
sh (
928+
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh",
929+
label: 'Build Hexagon API',
930+
)
931+
sh (
932+
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
933+
label: 'Run Hexagon tests',
934+
)
935+
})
936+
}
937+
} finally {
938+
junit 'build/pytest-results/*.xml'
939+
}
940+
}
941+
}
942+
} else {
943+
Utils.markStageSkippedForConditional('test: Hexagon 3 of 4')
944+
}
945+
},
946+
'test: Hexagon 4 of 4': {
947+
if (!skip_ci && is_docs_only_build != 1) {
948+
node('CPU') {
949+
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
950+
try {
951+
init_git()
952+
timeout(time: max_time, unit: 'MINUTES') {
953+
withEnv([
954+
'PLATFORM=hexagon',
955+
'TVM_NUM_SHARDS=4',
956+
'TVM_SHARD_INDEX=3'], {
957+
unpack_lib('hexagon', tvm_lib)
958+
ci_setup(ci_hexagon)
959+
sh (
960+
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh",
961+
label: 'Build Hexagon API',
962+
)
963+
sh (
964+
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
965+
label: 'Run Hexagon tests',
966+
)
967+
})
968+
}
969+
} finally {
970+
junit 'build/pytest-results/*.xml'
971+
}
972+
}
973+
}
974+
} else {
975+
Utils.markStageSkippedForConditional('test: Hexagon 4 of 4')
868976
}
869977
},
870978
'test: QEMU': {
@@ -1200,6 +1308,9 @@ stage('Test') {
12001308
},
12011309
)
12021310
}
1311+
}
1312+
1313+
test()
12031314

12041315
/*
12051316
stage('Build packages') {

‎jenkins/Jenkinsfile.j2‎

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ upstream_revision = null
8989
docker_run = 'docker/bash.sh --env CI --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM'
9090
docker_build = 'docker/build.sh'
9191
// timeout in minutes
92-
max_time = 240
92+
max_time = 120
9393
rebuild_docker_images = false
9494

9595
def per_exec_ws(folder) {
@@ -193,7 +193,7 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) {
193193

194194
cancel_previous_build()
195195

196-
196+
def lint() {
197197
stage('Lint') {
198198
node('CPU') {
199199
timeout(time: max_time, unit: 'MINUTES') {
@@ -249,6 +249,11 @@ stage('Lint') {
249249
}
250250
}
251251
}
252+
}
253+
254+
// This has to be extracted into a method due to JVM limitations on the size of
255+
// a method (so the code can't all be inlined)
256+
lint()
252257

253258
def build_image(image_name) {
254259
hash = sh(
@@ -462,6 +467,7 @@ def cpp_unittest(image) {
462467
)
463468
}
464469

470+
def build() {
465471
stage('Build') {
466472
environment {
467473
SKIP_SLOW_TESTS = "${skip_slow_tests}"
@@ -602,13 +608,16 @@ stage('Build') {
602608
}
603609
}
604610
}
611+
}
612+
build()
605613

614+
def test() {
606615
stage('Test') {
607616
environment {
608617
SKIP_SLOW_TESTS = "${skip_slow_tests}"
609618
}
610619
parallel(
611-
{% call m.sharded_test_step(
620+
{% call(shard_index) m.sharded_test_step(
612621
name="unittest: GPU",
613622
num_shards=2,
614623
node="GPU",
@@ -634,7 +643,7 @@ stage('Test') {
634643
label: 'Run Python GPU integration tests',
635644
)
636645
{% endcall %}
637-
{% call m.sharded_test_step(
646+
{% call(shard_index) m.sharded_test_step(
638647
name="integration: CPU",
639648
node="CPU",
640649
num_shards=2,
@@ -663,7 +672,7 @@ stage('Test') {
663672
label: 'Run VTA tests in TSIM',
664673
)
665674
{% endcall %}
666-
{% call m.sharded_test_step(
675+
{% call(shard_index) m.sharded_test_step(
667676
name="python: i386",
668677
node="CPU",
669678
num_shards=2,
@@ -680,14 +689,17 @@ stage('Test') {
680689
)
681690
fsim_test(ci_i386)
682691
{% endcall %}
683-
{% call m.test_step(
692+
{% call(shard_index) m.sharded_test_step(
684693
name="test: Hexagon",
685694
node="CPU", ws="tvm/test-hexagon",
686695
platform="hexagon",
696+
num_shards=4,
687697
) %}
688698
unpack_lib('hexagon', tvm_lib)
689699
ci_setup(ci_hexagon)
690-
cpp_unittest(ci_hexagon)
700+
{% if shard_index == 1 %}
701+
cpp_unittest(ci_hexagon)
702+
{% endif %}
691703
sh (
692704
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh",
693705
label: 'Build Hexagon API',
@@ -736,7 +748,7 @@ stage('Test') {
736748
label: 'Run TOPI tests',
737749
)
738750
{% endcall %}
739-
{% call m.sharded_test_step(
751+
{% call(shard_index) m.sharded_test_step(
740752
name="integration: aarch64",
741753
num_shards=2,
742754
node="ARM", ws="tvm/ut-python-arm",
@@ -750,7 +762,7 @@ stage('Test') {
750762
label: 'Run CPU integration tests',
751763
)
752764
{% endcall %}
753-
{% call m.sharded_test_step(
765+
{% call(shard_index) m.sharded_test_step(
754766
name="topi: GPU",
755767
node="GPU",
756768
num_shards=2,
@@ -764,7 +776,7 @@ stage('Test') {
764776
label: 'Run TOPI tests',
765777
)
766778
{% endcall %}
767-
{% call m.sharded_test_step(
779+
{% call(shard_index) m.sharded_test_step(
768780
name="frontend: GPU", node="GPU",
769781
num_shards=3,
770782
ws="tvm/frontend-python-gpu",
@@ -824,6 +836,9 @@ stage('Test') {
824836
},
825837
)
826838
}
839+
}
840+
841+
test()
827842

828843
/*
829844
stage('Build packages') {

‎jenkins/macros.j2‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
'PLATFORM={{ platform }}',
3333
'TVM_NUM_SHARDS={{ num_shards }}',
3434
'TVM_SHARD_INDEX={{ shard_index - 1 }}'], {
35-
{{ caller() | trim | indent(width=12) }}
35+
{{ caller(shard_index) | trim | indent(width=12) }}
3636
})
3737
}
3838
} finally {

0 commit comments

Comments
 (0)