forked from dmlc/gluon-cv
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Jenkinsfile
135 lines (130 loc) · 6.12 KB
/
Jenkinsfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
max_time = 120
stage("Sanity Check") {
node {
ws('workspace/gluon-cv-lint') {
timeout(time: max_time, unit: 'MINUTES') {
checkout scm
sh """#!/bin/bash
set -e
conda env update -n gluon-cv-lint -f tests/pylint.yml --prune
conda activate gluon-cv-lint
conda list
make clean
make pylint
"""
}
}
}
}
stage("Unit Test") {
parallel 'Python 2': {
node('linux-gpu') {
ws('workspace/gluon-cv-py2') {
timeout(time: max_time, unit: 'MINUTES') {
checkout scm
VISIBLE_GPU=env.EXECUTOR_NUMBER.toInteger() % 8
sh """#!/bin/bash
# old pip packages won't be cleaned: https://github.com/conda/conda/issues/5887
# remove and create new env instead
conda env remove -n gluon_cv_py2_test -y
set -ex
conda env create -n gluon_cv_py2_test -f tests/py2.yml
conda env update -n gluon_cv_py2_test -f tests/py2.yml --prune
conda activate gluon_cv_py2_test
conda list
export CUDA_VISIBLE_DEVICES=${VISIBLE_GPU}
export KMP_DUPLICATE_LIB_OK=TRUE
make clean
# from https://stackoverflow.com/questions/19548957/can-i-force-pip-to-reinstall-the-current-version
pip install --upgrade --force-reinstall --no-deps .
env
export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64
export MPLBACKEND=Agg
export MXNET_CUDNN_AUTOTUNE_DEFAULT=0
nosetests --with-timer --timer-ok 5 --timer-warning 20 -x -v tests/unittests
nosetests --with-timer --timer-ok 5 --timer-warning 20 -x -v tests/model_zoo
"""
}
}
}
},
'Python 3': {
node('linux-gpu') {
ws('workspace/gluon-cv-py3') {
timeout(time: max_time, unit: 'MINUTES') {
checkout scm
VISIBLE_GPU=env.EXECUTOR_NUMBER.toInteger() % 8
sh """#!/bin/bash
conda env remove -n gluon_cv_py3_test -y
set -ex
# remove and create new env instead
conda env create -n gluon_cv_py3_test -f tests/py3.yml
conda env update -n gluon_cv_py3_test -f tests/py3.yml --prune
conda activate gluon_cv_py3_test
conda list
export CUDA_VISIBLE_DEVICES=${VISIBLE_GPU}
export KMP_DUPLICATE_LIB_OK=TRUE
make clean
# from https://stackoverflow.com/questions/19548957/can-i-force-pip-to-reinstall-the-current-version
pip install --upgrade --force-reinstall --no-deps .
env
export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64
export MPLBACKEND=Agg
export MXNET_CUDNN_AUTOTUNE_DEFAULT=0
nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v tests/unittests
nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v tests/model_zoo
rm -f coverage.svg
coverage-badge -o coverage.svg
if [[ ${env.BRANCH_NAME} == master ]]; then
aws s3 cp coverage.svg s3://gluon-cv.mxnet.io/coverage.svg --acl public-read --cache-control no-cache
echo "Uploaded coverage badge to http://gluon-cv.mxnet.io"
else
aws s3 cp coverage.svg s3://gluon-vision-staging/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/coverage.svg --acl public-read --cache-control no-cache
echo "Uploaded coverage badge to http://gluon-vision-staging.s3-website-us-west-2.amazonaws.com/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/coverage.svg"
fi
"""
}
}
}
}
}
stage("Build Docs") {
node('linux-gpu') {
ws('workspace/gluon-cv-docs') {
timeout(time: max_time, unit: 'MINUTES') {
checkout scm
VISIBLE_GPU=env.EXECUTOR_NUMBER.toInteger() % 8
sh """#!/bin/bash
conda env remove -n gluon_vision_docs -y
set -ex
conda env create -n gluon_vision_docs -f docs/build.yml
conda env update -n gluon_vision_docs -f docs/build.yml --prune
conda activate gluon_vision_docs
export PYTHONPATH=\${PWD}
export CUDA_VISIBLE_DEVICES=${VISIBLE_GPU}
env
export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64
git submodule update --init --recursive
git clean -fx
cd docs && make clean && make html
sed -i.bak 's/33\\,150\\,243/23\\,141\\,201/g' build/html/_static/material-design-lite-1.3.0/material.blue-deep_orange.min.css;
sed -i.bak 's/2196f3/178dc9/g' build/html/_static/sphinx_materialdesign_theme.css;
sed -i.bak 's/pre{padding:1rem;margin:1.5rem\\s0;overflow:auto;overflow-y:hidden}/pre{padding:1rem;margin:1.5rem 0;overflow:auto;overflow-y:scroll}/g' build/html/_static/sphinx_materialdesign_theme.css
if [[ ${env.BRANCH_NAME} == master ]]; then
aws s3 cp s3://gluon-cv.mxnet.io/coverage.svg build/html/coverage.svg
aws s3 sync --delete build/html/ s3://gluon-cv.mxnet.io/ --acl public-read --cache-control max-age=7200
aws s3 cp build/html/coverage.svg s3://gluon-cv.mxnet.io/coverage.svg --acl public-read --cache-control max-age=300
echo "Uploaded doc to http://gluon-cv.mxnet.io"
else
aws s3 cp s3://gluon-vision-staging/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/coverage.svg build/html/coverage.svg
aws s3 sync --delete build/html/ s3://gluon-vision-staging/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/ --acl public-read
echo "Uploaded doc to http://gluon-vision-staging.s3-website-us-west-2.amazonaws.com/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/index.html"
fi
"""
if (env.BRANCH_NAME.startsWith("PR-")) {
pullRequest.comment("Job ${env.BRANCH_NAME}-${env.BUILD_NUMBER} is done. \nDocs are uploaded to http://gluon-vision-staging.s3-website-us-west-2.amazonaws.com/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/index.html \nCode coverage of this PR:  vs. Master: ")
}
}
}
}
}