From 8dc4c285d5e57ac4932df6cf0780d87fc61ce881 Mon Sep 17 00:00:00 2001 From: perdasilva Date: Mon, 11 Mar 2019 09:57:31 +0100 Subject: [PATCH 1/4] Backports Windows CI pipeline fixes from master --- ci/build_windows.py | 6 +++--- ci/jenkins/Jenkins_steps.groovy | 32 ++++++++++++------------------ ci/jenkins/Jenkinsfile_windows_cpu | 4 ++-- ci/windows/test_py2_cpu.ps1 | 8 +++++--- ci/windows/test_py2_gpu.ps1 | 12 ++++++----- ci/windows/test_py3_cpu.ps1 | 8 +++++--- ci/windows/test_py3_gpu.ps1 | 12 ++++++----- tests/requirements.txt | 3 +++ 8 files changed, 45 insertions(+), 40 deletions(-) diff --git a/ci/build_windows.py b/ci/build_windows.py index 56769f7cdaf0..427e50f1cc0c 100755 --- a/ci/build_windows.py +++ b/ci/build_windows.py @@ -216,11 +216,11 @@ def main(): if system == 'Windows': logging.info("Detected Windows platform") if 'OpenBLAS_HOME' not in os.environ: - os.environ["OpenBLAS_HOME"] = "C:\\mxnet\\openblas" + os.environ["OpenBLAS_HOME"] = "C:\\Program Files\\OpenBLAS-v0.2.19" if 'OpenCV_DIR' not in os.environ: - os.environ["OpenCV_DIR"] = "C:\\mxnet\\opencv_vc14" + os.environ["OpenCV_DIR"] = "C:\\Program Files\\OpenCV-v3.4.1\\build" if 'CUDA_PATH' not in os.environ: - os.environ["CUDA_PATH"] = "C:\\CUDA\\v8.0" + os.environ["CUDA_PATH"] = "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.2" windows_build(args) elif system == 'Linux' or system == 'Darwin': diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 81b573a97b24..73c60290d32b 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -450,13 +450,11 @@ def compile_unix_amalgamation() { def compile_windows_cpu() { return ['Build CPU windows':{ node(NODE_WINDOWS_CPU) { - timeout(time: max_time, unit: 'MINUTES') { - ws('workspace/build-cpu') { - withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { - utils.init_git_win() - powershell 'python ci/build_windows.py -f WIN_CPU' - stash includes: 'windows_package.7z', name: 'windows_package_cpu' - } + ws('workspace/build-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git_win() + powershell 'py -3 ci/build_windows.py -f WIN_CPU' + stash includes: 'windows_package.7z', name: 'windows_package_cpu' } } } @@ -466,13 +464,11 @@ def compile_windows_cpu() { def compile_windows_gpu() { return ['Build GPU windows':{ node(NODE_WINDOWS_CPU) { - timeout(time: max_time, unit: 'MINUTES') { - ws('workspace/build-gpu') { - withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { + ws('workspace/build-gpu') { + timeout(time: max_time, unit: 'MINUTES') { utils.init_git_win() - powershell 'python ci/build_windows.py -f WIN_GPU' + powershell 'py -3 ci/build_windows.py -f WIN_GPU' stash includes: 'windows_package.7z', name: 'windows_package_gpu' - } } } } @@ -482,13 +478,11 @@ def compile_windows_gpu() { def compile_windows_gpu_mkldnn() { return ['Build GPU MKLDNN windows':{ node(NODE_WINDOWS_CPU) { - timeout(time: max_time, unit: 'MINUTES') { - ws('workspace/build-gpu') { - withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) { - utils.init_git_win() - powershell 'python ci/build_windows.py -f WIN_GPU_MKLDNN' - stash includes: 'windows_package.7z', name: 'windows_package_gpu_mkldnn' - } + ws('workspace/build-gpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git_win() + powershell 'py -3 ci/build_windows.py -f WIN_GPU_MKLDNN' + stash includes: 'windows_package.7z', name: 'windows_package_gpu_mkldnn' } } } diff --git a/ci/jenkins/Jenkinsfile_windows_cpu b/ci/jenkins/Jenkinsfile_windows_cpu index 9e70df38dca5..67a06116da1f 100644 --- a/ci/jenkins/Jenkinsfile_windows_cpu +++ b/ci/jenkins/Jenkinsfile_windows_cpu @@ -35,12 +35,12 @@ utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ custom_steps.compile_windows_cpu() - ]) + ]) utils.parallel_stage('Tests', [ custom_steps.test_windows_python2_cpu(), custom_steps.test_windows_python3_cpu() - ]) + ]) } , failure_handler: { diff --git a/ci/windows/test_py2_cpu.ps1 b/ci/windows/test_py2_cpu.ps1 index 702a2db90ed7..4cef0ea84c20 100644 --- a/ci/windows/test_py2_cpu.ps1 +++ b/ci/windows/test_py2_cpu.ps1 @@ -19,8 +19,10 @@ $env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll $env:PYTHONPATH=join-path $pwd.Path windows_package\python $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 -c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt -c:\Anaconda3\envs\py2\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') + +C:\Python27\Scripts\pip install -r tests\requirements.txt +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest if (! $?) { Throw ("Error running unittest") } -c:\Anaconda3\envs\py2\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train if (! $?) { Throw ("Error running train tests") } diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1 index 0cf2717fa208..a113ee003274 100644 --- a/ci/windows/test_py2_gpu.ps1 +++ b/ci/windows/test_py2_gpu.ps1 @@ -19,12 +19,14 @@ $env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll $env:PYTHONPATH=join-path $pwd.Path windows_package\python $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 -c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt -c:\Anaconda3\envs\py2\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') + +C:\Python27\Scripts\pip install -r tests\requirements.txt +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest if (! $?) { Throw ("Error running unittest") } -c:\Anaconda3\envs\py2\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py if (! $?) { Throw ("Error running tests") } -c:\Anaconda3\envs\py2\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py if (! $?) { Throw ("Error running tests") } -c:\Anaconda3\envs\py2\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error tests\python\train +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error tests\python\train if (! $?) { Throw ("Error running tests") } diff --git a/ci/windows/test_py3_cpu.ps1 b/ci/windows/test_py3_cpu.ps1 index a7774a63dd86..9d838c26228d 100644 --- a/ci/windows/test_py3_cpu.ps1 +++ b/ci/windows/test_py3_cpu.ps1 @@ -19,8 +19,10 @@ $env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll $env:PYTHONPATH=join-path $pwd.Path windows_package\python $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 -c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt -c:\Anaconda3\envs\py3\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') + +C:\Python37\Scripts\pip install -r tests\requirements.txt +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest if (! $?) { Throw ("Error running unittest") } -c:\Anaconda3\envs\py3\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train if (! $?) { Throw ("Error running train tests") } diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1 index f9955ef6473f..5492538174cf 100644 --- a/ci/windows/test_py3_gpu.ps1 +++ b/ci/windows/test_py3_gpu.ps1 @@ -19,12 +19,14 @@ $env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll $env:PYTHONPATH=join-path $pwd.Path windows_package\python $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 -c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt -c:\Anaconda3\envs\py3\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') + +C:\Python37\Scripts\pip install -r tests\requirements.txt +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest if (! $?) { Throw ("Error running unittest") } -c:\Anaconda3\envs\py3\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py if (! $?) { Throw ("Error running tests") } -c:\Anaconda3\envs\py3\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py if (! $?) { Throw ("Error running tests") } -c:\Anaconda3\envs\py3\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train if (! $?) { Throw ("Error running tests") } diff --git a/tests/requirements.txt b/tests/requirements.txt index 3ca696b288c9..d58874084d7b 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,3 +2,6 @@ mock nose nose-timer +ipython +numpy +scipy From 1e39c1c83a251463c0786408ce4f7f9f2e379927 Mon Sep 17 00:00:00 2001 From: Per Goncalves da Silva Date: Wed, 13 Mar 2019 16:34:59 +0100 Subject: [PATCH 2/4] Updates Jenkins file to use python 3 when calling build_windows.py --- Jenkinsfile | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a9051db7b64e..03ad7d88acf6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -390,13 +390,11 @@ core_logic: { }, 'Build CPU windows':{ node(NODE_WINDOWS_CPU) { - timeout(time: max_time, unit: 'MINUTES') { - ws('workspace/build-cpu') { - withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { - utils.init_git_win() - powershell 'python ci/build_windows.py -f WIN_CPU' - stash includes: 'windows_package.7z', name: 'windows_package_cpu' - } + ws('workspace/build-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git_win() + powershell 'py -3 ci/build_windows.py -f WIN_CPU' + stash includes: 'windows_package.7z', name: 'windows_package_cpu' } } } @@ -404,26 +402,22 @@ core_logic: { 'Build GPU windows':{ node(NODE_WINDOWS_CPU) { - timeout(time: max_time, unit: 'MINUTES') { - ws('workspace/build-gpu') { - withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { + ws('workspace/build-gpu') { + timeout(time: max_time, unit: 'MINUTES') { utils.init_git_win() - powershell 'python ci/build_windows.py -f WIN_GPU' + powershell 'py -3 ci/build_windows.py -f WIN_GPU' stash includes: 'windows_package.7z', name: 'windows_package_gpu' - } } } } }, 'Build GPU MKLDNN windows':{ node(NODE_WINDOWS_CPU) { - timeout(time: max_time, unit: 'MINUTES') { - ws('workspace/build-gpu') { - withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) { - utils.init_git_win() - powershell 'python ci/build_windows.py -f WIN_GPU_MKLDNN' - stash includes: 'windows_package.7z', name: 'windows_package_gpu_mkldnn' - } + ws('workspace/build-gpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git_win() + powershell 'py -3 ci/build_windows.py -f WIN_GPU_MKLDNN' + stash includes: 'windows_package.7z', name: 'windows_package_gpu_mkldnn' } } } From cbc3f7970b99f39744a11b9553d29610caf0a20d Mon Sep 17 00:00:00 2001 From: perdasilva Date: Thu, 7 Mar 2019 20:05:55 +0100 Subject: [PATCH 3/4] Installs qemu pip requirements from qemu requirements file (#14355) --- ci/docker/qemu/runtime_functions.py | 4 ++-- ci/docker/qemu/vmcontrol.py | 1 + ci/qemu/README.md | 4 ++++ ci/qemu/test_requirements.txt | 3 +++ tests/requirements.txt | 1 + 5 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 ci/qemu/test_requirements.txt diff --git a/ci/docker/qemu/runtime_functions.py b/ci/docker/qemu/runtime_functions.py index 8b8e5acb503c..5a57cb8dae6a 100755 --- a/ci/docker/qemu/runtime_functions.py +++ b/ci/docker/qemu/runtime_functions.py @@ -77,8 +77,8 @@ def run_ut_python3_qemu_internal(): logging.info("=== NOW Running inside QEMU ===") logging.info("PIP Installing %s", pkg) check_call(['sudo', 'pip3', 'install', pkg]) - logging.info("PIP Installing mxnet/tests/requirements.txt") - check_call(['sudo', 'pip3', 'install', '-r', 'mxnet/tests/requirements.txt']) + logging.info("PIP Installing mxnet/test_requirements.txt") + check_call(['sudo', 'pip3', 'install', '-r', 'mxnet/test_requirements.txt']) logging.info("Running tests in mxnet/tests/python/unittest/") check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file', 'nosetests_unittest.xml', '--verbose', 'mxnet/tests/python/unittest/test_engine.py']) # Example to run a single unit test: diff --git a/ci/docker/qemu/vmcontrol.py b/ci/docker/qemu/vmcontrol.py index a7e8c0ff0122..a5b025bb6cae 100644 --- a/ci/docker/qemu/vmcontrol.py +++ b/ci/docker/qemu/vmcontrol.py @@ -187,6 +187,7 @@ def qemu_provision(ssh_port=QEMU_SSH_PORT): qemu_rsync(ssh_port, '/work/runtime_functions.py','') qemu_rsync(ssh_port, '/work/vmcontrol.py','') qemu_rsync(ssh_port, 'mxnet/tests', 'mxnet') + qemu_rsync(ssh_port, 'mxnet/ci/qemu/test_requirements.txt', 'mxnet/test_requirements.txt') logging.info("Provisioning completed successfully.") diff --git a/ci/qemu/README.md b/ci/qemu/README.md index 498f8b7a8739..4beca4a03690 100644 --- a/ci/qemu/README.md +++ b/ci/qemu/README.md @@ -86,3 +86,7 @@ pip3 install -r mxnet_requirements.txt To access qemu control console from tmux: `ctrl-a a c` + +# CI and Testing + +Formally, [runtime_functions.py](https://github.com/apache/incubator-mxnet/blob/master/ci/docker/qemu/runtime_functions.py) would [run](https://github.com/apache/incubator-mxnet/blob/8beea18e3d9835f90b59d3f9de8f9945ac819423/ci/docker/qemu/runtime_functions.py#L81) *pip install -r [mxnet/tests/requirements.txt](https://github.com/apache/incubator-mxnet/blob/master/tests/requirements.txt)*. If the requirements change, there can be an unfortunate side-effect that there are no wheel files for Raspberry Pi for the new requirement. This would trigger a build from source on the emulator, which can take a long time and cause job timeouts. Therefore, we no longer install the `tests/requirements.txt` requirements, but rather rely on [test_requirements.txt](https://github.com/apache/incubator-mxnet/blob/master/ci/qemu/test_requirements.txt) to maintain the requirements for the qemu tests. Should any requirements changes lead to a job time out, it is incumbent on the submitter to update the image to include the requirement and unblock ci. diff --git a/ci/qemu/test_requirements.txt b/ci/qemu/test_requirements.txt new file mode 100644 index 000000000000..77037d89c673 --- /dev/null +++ b/ci/qemu/test_requirements.txt @@ -0,0 +1,3 @@ +mock +nose +nose-timer \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt index d58874084d7b..e16b764d2746 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,5 @@ # Requirements for tests, those are installed before running on the virtualenv +# Requirements for tests run within the qemu requirement see ci/qemu/test_requirements.txt mock nose nose-timer From ee491902e1928c7404657dffd14d595c6bdfc572 Mon Sep 17 00:00:00 2001 From: JackieWu Date: Thu, 14 Mar 2019 07:58:58 +0800 Subject: [PATCH 4/4] remove a white space --- ci/docker/qemu/runtime_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/qemu/runtime_functions.py b/ci/docker/qemu/runtime_functions.py index 5a57cb8dae6a..9c137f6c32f2 100755 --- a/ci/docker/qemu/runtime_functions.py +++ b/ci/docker/qemu/runtime_functions.py @@ -77,7 +77,7 @@ def run_ut_python3_qemu_internal(): logging.info("=== NOW Running inside QEMU ===") logging.info("PIP Installing %s", pkg) check_call(['sudo', 'pip3', 'install', pkg]) - logging.info("PIP Installing mxnet/test_requirements.txt") + logging.info("PIP Installing mxnet/test_requirements.txt") check_call(['sudo', 'pip3', 'install', '-r', 'mxnet/test_requirements.txt']) logging.info("Running tests in mxnet/tests/python/unittest/") check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file', 'nosetests_unittest.xml', '--verbose', 'mxnet/tests/python/unittest/test_engine.py'])