From 1fed192efedc50eb8c65b8f03015dd2887d68130 Mon Sep 17 00:00:00 2001 From: perdasilva Date: Tue, 19 Mar 2019 18:08:48 +0100 Subject: [PATCH] Cherry-picks changes from #14336 to fix window ci pipeline (#14469) --- Jenkinsfile | 18 +++++++++--------- ci/build_windows.py | 6 +++--- ci/windows/test_py2_cpu.ps1 | 8 +++++--- ci/windows/test_py2_gpu.ps1 | 12 +++++++----- ci/windows/test_py3_cpu.ps1 | 8 +++++--- ci/windows/test_py3_gpu.ps1 | 12 +++++++----- tests/python/gpu/test_operator_gpu.py | 2 +- tests/requirements.txt | 4 ++++ 8 files changed, 41 insertions(+), 29 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0e4aa199a6c7..67926f8a6c52 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -317,9 +317,9 @@ core_logic: { node(NODE_WINDOWS_CPU) { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-cpu') { - withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { + timeout(time: max_time, unit: 'MINUTES') { utils.init_git_win() - powershell 'python ci/build_windows.py -f WIN_CPU' + powershell 'py -3 ci/build_windows.py -f WIN_CPU' stash includes: 'windows_package.7z', name: 'windows_package_cpu' } } @@ -331,10 +331,10 @@ core_logic: { node(NODE_WINDOWS_CPU) { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-gpu') { - withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { - utils.init_git_win() - powershell 'python ci/build_windows.py -f WIN_GPU' - stash includes: 'windows_package.7z', name: 'windows_package_gpu' + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git_win() + powershell 'py -3 ci/build_windows.py -f WIN_GPU' + stash includes: 'windows_package.7z', name: 'windows_package_gpu' } } } @@ -344,9 +344,9 @@ core_logic: { node(NODE_WINDOWS_CPU) { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-gpu') { - withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) { + timeout(time: max_time, unit: 'MINUTES') { utils.init_git_win() - powershell 'python ci/build_windows.py -f WIN_GPU_MKLDNN' + powershell 'py -3 ci/build_windows.py -f WIN_GPU_MKLDNN' stash includes: 'windows_package.7z', name: 'windows_package_gpu_mkldnn' } } @@ -935,4 +935,4 @@ failure_handler: { emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' } } -) +) \ No newline at end of file diff --git a/ci/build_windows.py b/ci/build_windows.py index 5eca58db7b74..a811f985b65c 100755 --- a/ci/build_windows.py +++ b/ci/build_windows.py @@ -232,11 +232,11 @@ def main(): if system == 'Windows': logging.info("Detected Windows platform") if 'OpenBLAS_HOME' not in os.environ: - os.environ["OpenBLAS_HOME"] = "C:\\mxnet\\openblas" + os.environ["OpenBLAS_HOME"] = "C:\\Program Files\\OpenBLAS-v0.2.19" if 'OpenCV_DIR' not in os.environ: - os.environ["OpenCV_DIR"] = "C:\\mxnet\\opencv_vc14" + os.environ["OpenCV_DIR"] = "C:\\Program Files\\OpenCV-v3.4.1\\build" if 'CUDA_PATH' not in os.environ: - os.environ["CUDA_PATH"] = "C:\\CUDA\\v8.0" + os.environ["CUDA_PATH"] = "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.2" windows_build(args) elif system == 'Linux' or system == 'Darwin': diff --git a/ci/windows/test_py2_cpu.ps1 b/ci/windows/test_py2_cpu.ps1 index aa38b81e392e..4cef0ea84c20 100644 --- a/ci/windows/test_py2_cpu.ps1 +++ b/ci/windows/test_py2_cpu.ps1 @@ -19,8 +19,10 @@ $env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll $env:PYTHONPATH=join-path $pwd.Path windows_package\python $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 -c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt -c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') + +C:\Python27\Scripts\pip install -r tests\requirements.txt +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest if (! $?) { Throw ("Error running unittest") } -c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_train.xml tests\python\train +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train if (! $?) { Throw ("Error running train tests") } diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1 index 5f8de5ac4f9e..a113ee003274 100644 --- a/ci/windows/test_py2_gpu.ps1 +++ b/ci/windows/test_py2_gpu.ps1 @@ -19,12 +19,14 @@ $env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll $env:PYTHONPATH=join-path $pwd.Path windows_package\python $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 -c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt -c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') + +C:\Python27\Scripts\pip install -r tests\requirements.txt +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest if (! $?) { Throw ("Error running unittest") } -c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py if (! $?) { Throw ("Error running tests") } -c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py if (! $?) { Throw ("Error running tests") } -c:\Anaconda3\envs\py2\python.exe -m nose -v tests\python\train +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error tests\python\train if (! $?) { Throw ("Error running tests") } diff --git a/ci/windows/test_py3_cpu.ps1 b/ci/windows/test_py3_cpu.ps1 index 0dd48de26b32..9d838c26228d 100644 --- a/ci/windows/test_py3_cpu.ps1 +++ b/ci/windows/test_py3_cpu.ps1 @@ -19,8 +19,10 @@ $env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll $env:PYTHONPATH=join-path $pwd.Path windows_package\python $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 -c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt -c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') + +C:\Python37\Scripts\pip install -r tests\requirements.txt +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest if (! $?) { Throw ("Error running unittest") } -c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_train.xml tests\python\train +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train if (! $?) { Throw ("Error running train tests") } diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1 index 4a0feb1ede88..5492538174cf 100644 --- a/ci/windows/test_py3_gpu.ps1 +++ b/ci/windows/test_py3_gpu.ps1 @@ -19,12 +19,14 @@ $env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll $env:PYTHONPATH=join-path $pwd.Path windows_package\python $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 -c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt -c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') + +C:\Python37\Scripts\pip install -r tests\requirements.txt +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest if (! $?) { Throw ("Error running unittest") } -c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py if (! $?) { Throw ("Error running tests") } -c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py if (! $?) { Throw ("Error running tests") } -c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_train.xml tests\python\train +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train if (! $?) { Throw ("Error running tests") } diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 3d6654d017bb..3f09ca19d857 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -1923,4 +1923,4 @@ def test_context_num_gpus(): if __name__ == '__main__': import nose - nose.runmodule() + nose.runmodule() \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt index 0eca73fbb02a..d58874084d7b 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,7 @@ # Requirements for tests, those are installed before running on the virtualenv mock nose +nose-timer +ipython +numpy +scipy