From b4545eae1aeaccdca804b40f93c559524b6bd06e Mon Sep 17 00:00:00 2001 From: Philip Cho Date: Sun, 12 May 2019 22:40:13 -0700 Subject: [PATCH 01/12] Fix #4462: Use /MT flag consistently for MSVC target --- CMakeLists.txt | 7 ++++++- cmake/Utils.cmake | 9 +++++++++ src/CMakeLists.txt | 8 +++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b8012476460..7b27a3f2d51b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,6 @@ if (MSVC) endif (MSVC) set_default_configuration_release() -msvc_use_static_runtime() #-- Options option(BUILD_C_DOC "Build documentation for C APIs using Doxygen." OFF) @@ -67,6 +66,7 @@ if (USE_CUDA) endif (USE_CUDA) # dmlc-core +msvc_use_static_runtime() add_subdirectory(${PROJECT_SOURCE_DIR}/dmlc-core) set_target_properties(dmlc PROPERTIES CXX_STANDARD 11 @@ -214,3 +214,8 @@ if (GOOGLE_TEST) PROPERTIES PASS_REGULAR_EXPRESSION ".*test-rmse:0.087.*") endif (GOOGLE_TEST) + +# For MSVC: Call msvc_use_static_runtime() once again to completely +# replace /MD with /MT. See https://github.com/dmlc/xgboost/issues/4462 +# for issues caused by mixing of /MD and /MT flags +msvc_use_static_runtime() diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake index d9b38dc8a8b4..8922fd0f08d9 100644 --- a/cmake/Utils.cmake +++ b/cmake/Utils.cmake @@ -17,6 +17,10 @@ endfunction(auto_source_group) function(msvc_use_static_runtime) if(MSVC) set(variables + CMAKE_C_FLAGS_DEBUG + CMAKE_C_FLAGS_MINSIZEREL + CMAKE_C_FLAGS_RELEASE + CMAKE_C_FLAGS_RELWITHDEBINFO CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELEASE @@ -29,6 +33,7 @@ function(msvc_use_static_runtime) endif() endforeach() set(variables + CMAKE_CUDA_FLAGS CMAKE_CUDA_FLAGS_DEBUG CMAKE_CUDA_FLAGS_MINSIZEREL CMAKE_CUDA_FLAGS_RELEASE @@ -39,6 +44,10 @@ function(msvc_use_static_runtime) string(REGEX REPLACE "-MD" "-MT" ${variable} "${${variable}}") set(${variable} "${${variable}}" PARENT_SCOPE) endif() + if(${variable} MATCHES "/MD") + string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}") + set(${variable} "${${variable}}" PARENT_SCOPE) + endif() endforeach() endif() endfunction(msvc_use_static_runtime) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2886706d2058..c053321b67e1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -42,7 +42,7 @@ if (USE_CUDA) $<$:--expt-extended-lambda> $<$:--expt-relaxed-constexpr> $<$:-lineinfo> - $<$:--std=c++11> + $<$>,$>:--std=c++11> $<$:${GEN_CODE}>) if (USE_NCCL) @@ -113,4 +113,10 @@ if (USE_OPENMP) set(LINKED_LIBRARIES_PRIVATE "${LINKED_LIBRARIES_PRIVATE};${SRC_LIBS}" PARENT_SCOPE) endif (OpenMP_CXX_FOUND OR OPENMP_FOUND) endif (USE_OPENMP) + +# For MSVC: Call msvc_use_static_runtime() once again to completely +# replace /MD with /MT. See https://github.com/dmlc/xgboost/issues/4462 +# for issues caused by mixing of /MD and /MT flags +msvc_use_static_runtime() + #-- End object library From 87ea09728c4eb205aba03b968b7e079f1bac7621 Mon Sep 17 00:00:00 2001 From: Philip Cho Date: Sun, 12 May 2019 23:29:15 -0700 Subject: [PATCH 02/12] First attempt at Windows CI --- Jenkinsfile-win64 | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 Jenkinsfile-win64 diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64 new file mode 100644 index 000000000000..f822916fcec6 --- /dev/null +++ b/Jenkinsfile-win64 @@ -0,0 +1,58 @@ +#!/usr/bin/groovy +// -*- mode: groovy -*- + +/* Jenkins pipeline for Windows AMD64 target */ + +pipeline { + agent none + // Build stages + stages { + stage('Get sources') { + agent { label 'win64' } + steps { + script { + checkoutSrcs() + } + stash name: 'srcs' + milestone ordinal: 1 + } + } + stage('Build') { + agent none + steps { + script { + parallel ([ + 'build-win64': { BuildWin64() } + ]) + } + milestone ordinal: 2 + } + } + } +} + +// check out source code from git +def checkoutSrcs() { + retry(5) { + try { + timeout(time: 2, unit: 'MINUTES') { + checkout scm + sh 'git submodule update --init' + } + } catch (exc) { + deleteDir() + error "Failed to fetch source codes" + } + } +} + +def BuildWin64() { + node('win64') { + unstash name: 'srcs' + echo "Building XGBoost for Windows AMD64 target..." + sh """ + ECHO "Good Morning" + """ + deleteDir() + } +} From ccaa96759c6ebd24270bdc9aae2f5e66f850c93e Mon Sep 17 00:00:00 2001 From: Philip Cho Date: Sun, 12 May 2019 23:42:09 -0700 Subject: [PATCH 03/12] Distinguish stages in Linux and Windows pipelines --- Jenkinsfile | 8 ++++---- Jenkinsfile-win64 | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d1ce168ec3cb..0fbbcd3db822 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,7 +25,7 @@ pipeline { // Build stages stages { - stage('Get sources') { + stage('Jenkins Linux: Get sources') { agent { label 'linux && cpu' } steps { script { @@ -35,7 +35,7 @@ pipeline { milestone ordinal: 1 } } - stage('Formatting Check') { + stage('Jenkins Linux: Formatting Check') { agent none steps { script { @@ -49,7 +49,7 @@ pipeline { milestone ordinal: 2 } } - stage('Build') { + stage('Jenkins Linux: Build') { agent none steps { script { @@ -65,7 +65,7 @@ pipeline { milestone ordinal: 3 } } - stage('Test') { + stage('Jenkins Linux: Test') { agent none steps { script { diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64 index f822916fcec6..d2c1d97be187 100644 --- a/Jenkinsfile-win64 +++ b/Jenkinsfile-win64 @@ -7,7 +7,7 @@ pipeline { agent none // Build stages stages { - stage('Get sources') { + stage('Jenkins Win64: Get sources') { agent { label 'win64' } steps { script { @@ -17,7 +17,7 @@ pipeline { milestone ordinal: 1 } } - stage('Build') { + stage('Jenkins Win64: Build') { agent none steps { script { From 8ec24c222a524630d979e4b9bb01241a301c9bd5 Mon Sep 17 00:00:00 2001 From: Philip Cho Date: Sun, 12 May 2019 23:42:21 -0700 Subject: [PATCH 04/12] Try running CMake in Windows pipeline --- Jenkinsfile-win64 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64 index d2c1d97be187..d5d31855e0ed 100644 --- a/Jenkinsfile-win64 +++ b/Jenkinsfile-win64 @@ -51,7 +51,9 @@ def BuildWin64() { unstash name: 'srcs' echo "Building XGBoost for Windows AMD64 target..." sh """ - ECHO "Good Morning" + mkdir build + cd build + cmake .. -G"Visual Studio 15 2017 Win64" -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON """ deleteDir() } From cbe6329c31c1d6dd9cdc6b337ab2370b56bd462a Mon Sep 17 00:00:00 2001 From: Philip Cho Date: Sun, 12 May 2019 23:52:05 -0700 Subject: [PATCH 05/12] Add build step --- Jenkinsfile-win64 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64 index d5d31855e0ed..d3024ad05bb9 100644 --- a/Jenkinsfile-win64 +++ b/Jenkinsfile-win64 @@ -54,7 +54,15 @@ def BuildWin64() { mkdir build cd build cmake .. -G"Visual Studio 15 2017 Win64" -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON + "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" amd64 + msbuild xgboost.sln /m /p:Configuration=Release -fl -flp:logfile=MSBuild.log;verbosity=diagnostic + cd ..\\python-package + conda activate + python setup.py bdist_wheel --universal """ + stash name: 'xgboost_win_whl', includes: 'python-package/dist/*.whl' + archiveArtifacts artifacts: "python-package/dist/*.whl", allowEmptyArchive: true + archiveArtifacts artifacts: "build/MSBuild.log", allowEmptyArchive: true deleteDir() } } From ec3a148569f80ceb3dbefe38fb0bfdd868ec0359 Mon Sep 17 00:00:00 2001 From: Philip Cho Date: Sun, 12 May 2019 23:58:06 -0700 Subject: [PATCH 06/12] Specify full path of msbuild --- Jenkinsfile-win64 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64 index d3024ad05bb9..8dcf9de88428 100644 --- a/Jenkinsfile-win64 +++ b/Jenkinsfile-win64 @@ -54,8 +54,7 @@ def BuildWin64() { mkdir build cd build cmake .. -G"Visual Studio 15 2017 Win64" -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON - "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" amd64 - msbuild xgboost.sln /m /p:Configuration=Release -fl -flp:logfile=MSBuild.log;verbosity=diagnostic + "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release -fl -flp:logfile=MSBuild.log;verbosity=diagnostic cd ..\\python-package conda activate python setup.py bdist_wheel --universal From 8450f059d7703ff869244f3b278a835b6d1f9d54 Mon Sep 17 00:00:00 2001 From: Philip Cho Date: Mon, 13 May 2019 00:02:36 -0700 Subject: [PATCH 07/12] Use CMD (bat) instead of Git Bash --- Jenkinsfile-win64 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64 index 8dcf9de88428..2d758e80775c 100644 --- a/Jenkinsfile-win64 +++ b/Jenkinsfile-win64 @@ -50,11 +50,12 @@ def BuildWin64() { node('win64') { unstash name: 'srcs' echo "Building XGBoost for Windows AMD64 target..." - sh """ + bat """ mkdir build cd build cmake .. -G"Visual Studio 15 2017 Win64" -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON - "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release -fl -flp:logfile=MSBuild.log;verbosity=diagnostic + "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" amd64 + msbuild xgboost.sln /m /p:Configuration=Release -fl -flp:logfile=MSBuild.log;verbosity=diagnostic cd ..\\python-package conda activate python setup.py bdist_wheel --universal From 13400a36e89b6d0aebaabe6fbbdee8d11a5d79f1 Mon Sep 17 00:00:00 2001 From: Philip Cho Date: Mon, 13 May 2019 00:06:25 -0700 Subject: [PATCH 08/12] Eschew vcvarsall.bat --- Jenkinsfile-win64 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64 index 2d758e80775c..d68546a0ba2e 100644 --- a/Jenkinsfile-win64 +++ b/Jenkinsfile-win64 @@ -54,8 +54,7 @@ def BuildWin64() { mkdir build cd build cmake .. -G"Visual Studio 15 2017 Win64" -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON - "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" amd64 - msbuild xgboost.sln /m /p:Configuration=Release -fl -flp:logfile=MSBuild.log;verbosity=diagnostic + "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release -fl -flp:logfile=MSBuild.log;verbosity=diagnostic cd ..\\python-package conda activate python setup.py bdist_wheel --universal From 38cb8cb46a7515ee31c62711f1d2cf733b6bf3f8 Mon Sep 17 00:00:00 2001 From: Philip Cho Date: Mon, 13 May 2019 00:18:35 -0700 Subject: [PATCH 09/12] Chain conda activate with Python wheel build command --- Jenkinsfile-win64 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64 index d68546a0ba2e..76c184ba66ca 100644 --- a/Jenkinsfile-win64 +++ b/Jenkinsfile-win64 @@ -56,8 +56,7 @@ def BuildWin64() { cmake .. -G"Visual Studio 15 2017 Win64" -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release -fl -flp:logfile=MSBuild.log;verbosity=diagnostic cd ..\\python-package - conda activate - python setup.py bdist_wheel --universal + conda activate && python setup.py bdist_wheel --universal """ stash name: 'xgboost_win_whl', includes: 'python-package/dist/*.whl' archiveArtifacts artifacts: "python-package/dist/*.whl", allowEmptyArchive: true From 7ab6efe69ba45838433acf499aaa7ada8176d9e5 Mon Sep 17 00:00:00 2001 From: Philip Cho Date: Mon, 13 May 2019 12:57:56 -0700 Subject: [PATCH 10/12] Split commands to detect failure --- Jenkinsfile-win64 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64 index 76c184ba66ca..dc55ddb7a091 100644 --- a/Jenkinsfile-win64 +++ b/Jenkinsfile-win64 @@ -54,8 +54,13 @@ def BuildWin64() { mkdir build cd build cmake .. -G"Visual Studio 15 2017 Win64" -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON + """ + bat """ + cd build "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release -fl -flp:logfile=MSBuild.log;verbosity=diagnostic - cd ..\\python-package + """ + bat """ + cd python-package conda activate && python setup.py bdist_wheel --universal """ stash name: 'xgboost_win_whl', includes: 'python-package/dist/*.whl' From 28dc21bc8f4b9ac995f4c72d8a0a5c815de063b5 Mon Sep 17 00:00:00 2001 From: Philip Cho Date: Mon, 13 May 2019 18:22:10 -0700 Subject: [PATCH 11/12] Try running MSBuild up to three times --- Jenkinsfile-win64 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64 index dc55ddb7a091..9c50ebd1ece0 100644 --- a/Jenkinsfile-win64 +++ b/Jenkinsfile-win64 @@ -57,7 +57,7 @@ def BuildWin64() { """ bat """ cd build - "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release -fl -flp:logfile=MSBuild.log;verbosity=diagnostic + "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release || "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release || "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release """ bat """ cd python-package @@ -65,7 +65,6 @@ def BuildWin64() { """ stash name: 'xgboost_win_whl', includes: 'python-package/dist/*.whl' archiveArtifacts artifacts: "python-package/dist/*.whl", allowEmptyArchive: true - archiveArtifacts artifacts: "build/MSBuild.log", allowEmptyArchive: true deleteDir() } } From 8077abbbf740bcfadc91a934af9b96056dee1c33 Mon Sep 17 00:00:00 2001 From: Philip Cho Date: Mon, 13 May 2019 20:46:36 -0700 Subject: [PATCH 12/12] Add back add_dependencies(), per @RAMitchell's suggestion --- CMakeLists.txt | 2 ++ Jenkinsfile-win64 | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b27a3f2d51b..bdaaa6661116 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,6 +139,8 @@ set_target_properties( set_output_directory(runxgboost ${PROJECT_SOURCE_DIR}) set_output_directory(xgboost ${PROJECT_SOURCE_DIR}/lib) +# Ensure these two targets do not build simultaneously, as they produce outputs with conflicting names +add_dependencies(xgboost runxgboost) #-- Installing XGBoost if (R_LIB) diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64 index 9c50ebd1ece0..831f9154c42e 100644 --- a/Jenkinsfile-win64 +++ b/Jenkinsfile-win64 @@ -57,7 +57,7 @@ def BuildWin64() { """ bat """ cd build - "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release || "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release || "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release + "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release """ bat """ cd python-package