diff --git a/ThirdPartyNotices.txt b/ThirdPartyNotices.txt index 176da7f186ca5..50a692aa24f67 100644 --- a/ThirdPartyNotices.txt +++ b/ThirdPartyNotices.txt @@ -2168,38 +2168,6 @@ See the [community structure document](http://docs.tvm.ai/contribute/community.h _____ -jemalloc - -Unless otherwise specified, files in the jemalloc source distribution are -subject to the following license: --------------------------------------------------------------------------------- -Copyright (C) 2002-2018 Jason Evans . -All rights reserved. -Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. -Copyright (C) 2009-2018 Facebook, Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright notice(s), - this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice(s), - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- - -_____ - FreeBSD: getopt.c file Copyright (c) 1987, 1993, 1994 diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index c031fc6830cf7..7d2a7f2bea618 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -59,7 +59,6 @@ option(onnxruntime_ENABLE_MEMLEAK_CHECKER "Experimental: Enable memory leak chec option(onnxruntime_USE_CUDA "Build with CUDA support" OFF) option(onnxruntime_ENABLE_CUDA_LINE_NUMBER_INFO "When building with CUDA support, generate device code line number information." OFF) option(onnxruntime_USE_OPENVINO "Build with OpenVINO support" OFF) -option(onnxruntime_USE_EIGEN_FOR_BLAS "Use eign for blas" ON) option(onnxruntime_USE_COREML "Build with CoreML support" OFF) option(onnxruntime_USE_NNAPI_BUILTIN "Build with builtin NNAPI lib for Android NNAPI support" OFF) option(onnxruntime_USE_RKNPU "Build with RKNPU support" OFF) @@ -161,6 +160,10 @@ if(onnxruntime_USE_VALGRIND AND NOT WIN32) add_definitions(-DRE2_ON_VALGRIND=1) endif() +if(WIN32) + string(APPEND CMAKE_CXX_FLAGS " /W3") +endif() + if (onnxruntime_ENABLE_NVTX_PROFILE) add_definitions(-DENABLE_NVTX_PROFILE=1) endif() @@ -643,18 +646,6 @@ if (onnxruntime_USE_CUDA AND "${onnxruntime_CUDNN_HOME}" STREQUAL "") message(FATAL_ERROR "onnxruntime_CUDNN_HOME required for onnxruntime_USE_CUDA") endif() -if (onnxruntime_USE_EIGEN_FOR_BLAS) - add_definitions(-DUSE_EIGEN_FOR_BLAS) -endif() - -if (onnxruntime_USE_OPENBLAS AND "${onnxruntime_OPENBLAS_HOME}" STREQUAL "" AND WIN32) - # On linux we assume blas is installed via 'apt-get install libopenblas-dev' - message(FATAL_ERROR "onnxruntime_OPENBLAS_HOME required for onnxruntime_USE_OPENBLAS") -endif() - -if (onnxruntime_USE_OPENBLAS AND onnxruntime_USE_EIGEN_FOR_BLAS) - message(FATAL_ERROR "use one of onnxruntime_USE_OPENBLAS, onnxruntime_USE_EIGEN_FOR_BLAS") -endif() get_filename_component(ONNXRUNTIME_ROOT "${ONNXRUNTIME_ROOT}" ABSOLUTE) get_filename_component(ORTTRAINING_ROOT "${ORTTRAINING_ROOT}" ABSOLUTE) @@ -701,11 +692,81 @@ set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} -DGSL_UNENFORCED_O include(eigen) -#onnxruntime_EXTERNAL_LIBRARIES could contain onnx, onnx_proto,libprotobuf, cuda/cudnn, jemalloc, -# dnnl/mklml, openblas, onnxruntime_codegen_tvm, tvm, nnvm_compiler and pthread +#onnxruntime_EXTERNAL_LIBRARIES could contain onnx, onnx_proto,libprotobuf, cuda/cudnn, +# dnnl/mklml, onnxruntime_codegen_tvm, tvm, nnvm_compiler and pthread # pthread is always at the last set(onnxruntime_EXTERNAL_LIBRARIES onnx onnx_proto ${PROTOBUF_LIB} re2::re2) + +set(onnxruntime_LINK_DIRS ) +if(onnxruntime_USE_CUDA) + #TODO: combine onnxruntime_CUDNN_HOME and onnxruntime_CUDA_HOME, assume they are the same + if (WIN32) + list(APPEND onnxruntime_LINK_DIRS ${onnxruntime_CUDNN_HOME}/lib/x64 ${onnxruntime_CUDA_HOME}/x64/lib64) + else() + list(APPEND onnxruntime_LINK_DIRS ${onnxruntime_CUDNN_HOME}/lib64 ${onnxruntime_CUDA_HOME}/lib64) + endif() +endif() + +function(onnxruntime_add_shared_library target_name) + add_library(${target_name} SHARED ${ARGN}) + target_link_directories(${target_name} PRIVATE ${onnxruntime_LINK_DIRS}) + if (MSVC) + target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") + target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /sdl>" "$<$>:/sdl>") + set_target_properties(${target_name} PROPERTIES VS_CA_EXCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") + else() + target_compile_definitions(${target_name} PUBLIC -DNSYNC_ATOMIC_CPP11) + target_include_directories(${target_name} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") + endif() + target_include_directories(${target_name} PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) + if(onnxruntime_ENABLE_LTO) + set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) + set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO TRUE) + set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_MINSIZEREL TRUE) + endif() +endfunction() + +#For plugins that are not linked into other targets but may be loaded dynamically at runtime using dlopen-like functionality. +function(onnxruntime_add_shared_library_module target_name) + add_library(${target_name} MODULE ${ARGN}) + target_link_directories(${target_name} PRIVATE ${onnxruntime_LINK_DIRS}) + if (MSVC) + target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") + target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /sdl>" "$<$>:/sdl>") + set_target_properties(${target_name} PROPERTIES VS_CA_EXCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") + else() + target_compile_definitions(${target_name} PUBLIC -DNSYNC_ATOMIC_CPP11) + target_include_directories(${target_name} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") + endif() + target_include_directories(${target_name} PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) + if(onnxruntime_ENABLE_LTO) + set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) + set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO TRUE) + set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_MINSIZEREL TRUE) + endif() +endfunction() + +#almost the same as the above function, except the first line of the body +function(onnxruntime_add_executable target_name) + add_executable(${target_name} ${ARGN}) + target_link_directories(${target_name} PRIVATE ${onnxruntime_LINK_DIRS}) + if (MSVC) + target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") + target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /sdl>" "$<$>:/sdl>") + set_target_properties(${target_name} PROPERTIES VS_CA_EXCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") + else() + target_compile_definitions(${target_name} PUBLIC -DNSYNC_ATOMIC_CPP11) + target_include_directories(${target_name} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") + endif() + target_include_directories(${target_name} PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) + if(onnxruntime_ENABLE_LTO) + set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) + set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO TRUE) + set_target_properties(${target_name} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_MINSIZEREL TRUE) + endif() +endfunction() + function(onnxruntime_add_include_to_target dst_target) foreach(src_target ${ARGN}) target_include_directories(${dst_target} PRIVATE $) @@ -890,6 +951,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Android" AND Onnxruntime_GCOV_COVERAGE) string(APPEND CMAKE_C_FLAGS " -g -O0 --coverage ") endif() +set(ORT_WARNING_FLAGS ) #Adjust warning flags if (WIN32) add_definitions(-DPLATFORM_WINDOWS -DNOGDI -DNOMINMAX -D_USE_MATH_DEFINES) @@ -898,27 +960,30 @@ if (WIN32) endif() # parallel build # These compiler opitions cannot be forwarded to NVCC, so cannot use add_compiler_options - string(APPEND CMAKE_CXX_FLAGS " /MP /W4") - string(APPEND CMAKE_C_FLAGS " /MP /W4") + string(APPEND CMAKE_CXX_FLAGS " /MP") + if(onnxruntime_USE_CUDA) + list(APPEND ORT_WARNING_FLAGS "/W3") + else() + list(APPEND ORT_WARNING_FLAGS "/W4") + endif() #Compiler bug, we should get such warnings. It will be fixed in a new VC release - string(APPEND CMAKE_CXX_FLAGS " /wd4127") + list(APPEND ORT_WARNING_FLAGS "/wd4127") # class needs to have dll-interface to be used by clients - string(APPEND CMAKE_CXX_FLAGS " /wd4251") + list(APPEND ORT_WARNING_FLAGS "/wd4251") # issued by thrust nonstandard extension used: nameless struct/union - string(APPEND CMAKE_CXX_FLAGS " /wd4201") + list(APPEND ORT_WARNING_FLAGS "/wd4201") if (onnxruntime_ENABLE_STATIC_ANALYSIS) - string(APPEND CMAKE_CXX_FLAGS - " /analyze:stacksize 131072" - # disable warning because there are many occurrences from test macros - " /wd6326 " # potential comparison of a constant with another constant - ) + list(APPEND ORT_WARNING_FLAGS "/analyze:stacksize 131072") + list(APPEND ORT_WARNING_FLAGS "/wd6326") # potential comparison of a constant with another constant + if(onnxruntime_USE_OPENMP) + list(APPEND ORT_WARNING_FLAGS "/wd6993") # Code analysis ignores OpenMP constructs + endif() endif() - # Treat warning as error if onnxruntime_DEV_MODE is ON # For cross-compiled ARM64 binaries, there are too many warnings to fix, hence ignore warnings for now if (onnxruntime_DEV_MODE AND NOT CMAKE_CROSSCOMPILING) # treat warnings as errors - string(APPEND CMAKE_CXX_FLAGS " /WX") + list(APPEND ORT_WARNING_FLAGS "/WX") foreach(type EXE STATIC SHARED) set(CMAKE_${type}_LINKER_FLAGS "${CMAKE_${type}_LINKER_FLAGS} /WX") endforeach() @@ -946,6 +1011,10 @@ if (WIN32) endif() endforeach() endif() + foreach(ORT_FLAG ${ORT_WARNING_FLAGS}) + string(APPEND CMAKE_CXX_FLAGS " ${ORT_FLAG}") + string(APPEND CMAKE_C_FLAGS " ${ORT_FLAG}") + endforeach() else() add_definitions(-DPLATFORM_POSIX) # Enable warning and enable space optimization in Linux @@ -991,19 +1060,6 @@ else() endif() endif() set(onnxruntime_DELAYLOAD_FLAGS "") -if (onnxruntime_USE_JEMALLOC) - if (onnxruntime_USE_MIMALLOC_STL_ALLOCATOR OR onnxruntime_USE_MIMALLOC_ARENA_ALLOCATOR) - message( FATAL_ERROR "You cannot specify both jemalloc and mimalloc." ) - endif() - - if (Win32) - message( FATAL_ERROR "Jemalloc is not supported on Windows." ) - endif() - include(jemalloc) - add_definitions(-DUSE_JEMALLOC=1) - list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${JEMALLOC_STATIC_LIBRARIES}) - list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES jemalloc) -endif() include_directories( ${ONNXRUNTIME_INCLUDE_DIR} @@ -1087,17 +1143,6 @@ if (onnxruntime_USE_VITISAI) endif() endif() -if (onnxruntime_USE_OPENBLAS) - add_definitions(-DUSE_OPENBLAS=1) - if (WIN32) - include_directories(${onnxruntime_OPENBLAS_HOME}) - list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${onnxruntime_OPENBLAS_HOME}/lib/libopenblas.lib) - else() - # on linux we assume blas is installed via 'apt-get install libopenblas-dev' - list(APPEND onnxruntime_EXTERNAL_LIBRARIES openblas) - endif() -endif() - configure_file(onnxruntime_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/onnxruntime_config.h) if (onnxruntime_USE_CUDA) @@ -1129,25 +1174,6 @@ if (onnxruntime_USE_CUDA) list(APPEND ONNXRUNTIME_CUDA_LIBRARIES cublas cudnn curand cufft) endif() - # CUDA_HOME and CUDNN_HOME may differ, so need to add both to the link directories - if (WIN32) - link_directories(${onnxruntime_CUDA_HOME}/lib/x64) - link_directories(${onnxruntime_CUDNN_HOME}/lib/x64) - - # delayload causes crash on exit, so disable for now - # please update cudaDelayLoadedLibs in Microsoft.ML.OnnxRuntime/SessionOptions.cs if you change delayload - #file(GLOB cuda_dll_paths "${onnxruntime_CUDA_HOME}/bin/cublas64_*" "${onnxruntime_CUDA_HOME}/bin/cudart64_*" "${onnxruntime_CUDA_HOME}/bin/curand64_*" "${onnxruntime_CUDA_HOME}/bin/cufft64_*") - #set(onnxruntime_DELAYLOAD_FLAGS "${onnxruntime_DELAYLOAD_FLAGS} /DELAYLOAD:cudnn64_8.dll") - #foreach(cuda_dll_path ${cuda_dll_paths}) - # get_filename_component(cuda_dll_file_name ${cuda_dll_path} NAME) - # set(onnxruntime_DELAYLOAD_FLAGS "${onnxruntime_DELAYLOAD_FLAGS} /DELAYLOAD:${cuda_dll_file_name}") - #endforeach(cuda_dll_path) - - else() - link_directories(${onnxruntime_CUDA_HOME}/lib64) - link_directories(${onnxruntime_CUDNN_HOME}/lib64) - endif() - list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${ONNXRUNTIME_CUDA_LIBRARIES}) if(NOT CMAKE_CUDA_ARCHITECTURES) if(CMAKE_LIBRARY_ARCHITECTURE STREQUAL "aarch64-linux-gnu") @@ -1368,6 +1394,7 @@ foreach(target_name onnxruntime_common onnxruntime_graph onnxruntime_framework o if (MSVC) target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /sdl>" "$<$>:/sdl>") + set_target_properties(${target_name} PROPERTIES VS_CA_EXCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") else() target_compile_definitions(${target_name} PUBLIC -DNSYNC_ATOMIC_CPP11) target_include_directories(${target_name} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") diff --git a/cmake/ConfigureVisualStudioCodeAnalysis.props b/cmake/ConfigureVisualStudioCodeAnalysis.props deleted file mode 100644 index bad2a87b930e7..0000000000000 --- a/cmake/ConfigureVisualStudioCodeAnalysis.props +++ /dev/null @@ -1,14 +0,0 @@ - - - - true - NativeRecommendedRules.ruleset - false - - $(SolutionDir);$(SolutionDir)..\..\..\cmake; - - diff --git a/cmake/EnableVisualStudioCodeAnalysis.props b/cmake/EnableVisualStudioCodeAnalysis.props deleted file mode 100644 index 9da9f0be46560..0000000000000 --- a/cmake/EnableVisualStudioCodeAnalysis.props +++ /dev/null @@ -1,24 +0,0 @@ - - - - true - NativeRecommendedRules.ruleset - - $(SolutionDir);$(MSBuildThisFileDirectory) - - - true - - - diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 64a4b6b64aea3..3b283767c589c 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -33,16 +33,15 @@ add_custom_command(OUTPUT ${SYMBOL_FILE} ${CMAKE_CURRENT_BINARY_DIR}/generated_s add_custom_target(onnxruntime_generate_def ALL DEPENDS ${SYMBOL_FILE} ${CMAKE_CURRENT_BINARY_DIR}/generated_source.c) if(WIN32) - add_library(onnxruntime SHARED + onnxruntime_add_shared_library(onnxruntime ${SYMBOL_FILE} "${ONNXRUNTIME_ROOT}/core/dll/dllmain.cc" "${ONNXRUNTIME_ROOT}/core/dll/onnxruntime.rc" ) else() - add_library(onnxruntime SHARED ${CMAKE_CURRENT_BINARY_DIR}/generated_source.c) + onnxruntime_add_shared_library(onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/generated_source.c) endif() -set_target_properties(onnxruntime PROPERTIES VERSION ${ORT_VERSION}) add_dependencies(onnxruntime onnxruntime_generate_def ${onnxruntime_EXTERNAL_DEPENDENCIES}) target_include_directories(onnxruntime PRIVATE ${ONNXRUNTIME_ROOT}) onnxruntime_add_include_to_target(onnxruntime) @@ -129,11 +128,10 @@ endif() set_property(TARGET onnxruntime APPEND_STRING PROPERTY LINK_FLAGS ${ONNXRUNTIME_SO_LINK_FLAG} ${onnxruntime_DELAYLOAD_FLAGS}) set_target_properties(onnxruntime PROPERTIES LINK_DEPENDS ${SYMBOL_FILE}) -if(onnxruntime_ENABLE_LTO) - set_target_properties(onnxruntime PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) - set_target_properties(onnxruntime PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO TRUE) - set_target_properties(onnxruntime PROPERTIES INTERPROCEDURAL_OPTIMIZATION_MINSIZEREL TRUE) -endif() + + +set_target_properties(onnxruntime PROPERTIES VERSION ${ORT_VERSION}) + install(TARGETS onnxruntime ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/cmake/onnxruntime_framework.cmake b/cmake/onnxruntime_framework.cmake index 06542f7437183..b40e91013bf2a 100644 --- a/cmake/onnxruntime_framework.cmake +++ b/cmake/onnxruntime_framework.cmake @@ -48,8 +48,3 @@ endif() install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/framework DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core) -if (WIN32) - # Add Code Analysis properties to enable C++ Core checks. Have to do it via a props file include. - set_target_properties(onnxruntime_framework PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/ConfigureVisualStudioCodeAnalysis.props) -endif() - diff --git a/cmake/onnxruntime_fuzz_test.cmake b/cmake/onnxruntime_fuzz_test.cmake index 381a08573bac0..b7cd95cb8693f 100644 --- a/cmake/onnxruntime_fuzz_test.cmake +++ b/cmake/onnxruntime_fuzz_test.cmake @@ -36,7 +36,7 @@ if (onnxruntime_FUZZ_ENABLED) "${SEC_FUZZ_ROOT}/src/test.cpp") # compile the executables - add_executable(onnxruntime_security_fuzz ${SEC_FUZ_SRC}) + onnxruntime_add_executable(onnxruntime_security_fuzz ${SEC_FUZ_SRC}) # compile with c++17 target_compile_features(onnxruntime_security_fuzz PUBLIC cxx_std_17) diff --git a/cmake/onnxruntime_java.cmake b/cmake/onnxruntime_java.cmake index afc28602ea9bb..54eb81f83306f 100644 --- a/cmake/onnxruntime_java.cmake +++ b/cmake/onnxruntime_java.cmake @@ -160,8 +160,11 @@ endif() # On Windows TARGET_LINKER_FILE_NAME is the .lib, TARGET_FILE_NAME is the .dll if (WIN32) - add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_LIB_DIR}/$) - add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_JNI_DIR}/$) + #Our static analysis plugin set /p:LinkCompiled=false + if(NOT onnxruntime_ENABLE_STATIC_ANALYSIS) + add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_LIB_DIR}/$) + add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_JNI_DIR}/$) + endif() else() add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_LIB_DIR}/$) add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_JNI_DIR}/$) diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 3c6cd8fbe3d21..30ea74419c933 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -265,13 +265,25 @@ if (onnxruntime_USE_CUDA) endif() add_library(onnxruntime_providers_cuda ${onnxruntime_providers_cuda_src}) - + + #target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler \"/analyze:stacksize 131072\">") + if (HAS_GUARD_CF) + target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler /guard:cf>") + endif() + if (HAS_QSPECTRE) + target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler /Qspectre>") + endif() + foreach(ORT_FLAG ${ORT_WARNING_FLAGS}) + target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler \"${ORT_FLAG}\">") + endforeach() if (UNIX) target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler -Wno-reorder>" "$<$>:-Wno-reorder>") target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler -Wno-error=sign-compare>" "$<$>:-Wno-error=sign-compare>") else() + #mutex.cuh(91): warning C4834: discarding return value of function with 'nodiscard' attribute + target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler /wd4834>") target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler /wd4127>") endif() onnxruntime_add_include_to_target(onnxruntime_providers_cuda onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf flatbuffers) @@ -335,7 +347,7 @@ if (onnxruntime_USE_TENSORRT OR onnxruntime_USE_DNNL OR onnxruntime_USE_OPENVINO ) source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_shared_cc_srcs}) - add_library(onnxruntime_providers_shared SHARED ${onnxruntime_providers_shared_cc_srcs}) + onnxruntime_add_shared_library(onnxruntime_providers_shared ${onnxruntime_providers_shared_cc_srcs}) set_target_properties(onnxruntime_providers_shared PROPERTIES FOLDER "ONNXRuntime") set_target_properties(onnxruntime_providers_shared PROPERTIES LINKER_LANGUAGE CXX) @@ -366,7 +378,7 @@ if (onnxruntime_USE_DNNL) ) source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_dnnl_cc_srcs}) - add_library(onnxruntime_providers_dnnl SHARED ${onnxruntime_providers_dnnl_cc_srcs}) + onnxruntime_add_shared_library_module(onnxruntime_providers_dnnl ${onnxruntime_providers_dnnl_cc_srcs}) target_link_directories(onnxruntime_providers_dnnl PRIVATE ${DNNL_LIB_DIR}) onnxruntime_add_include_to_target(onnxruntime_providers_dnnl onnxruntime_common onnx) # onnx needed for stl_backports.h add_dependencies(onnxruntime_providers_dnnl onnxruntime_providers_shared project_dnnl ${onnxruntime_EXTERNAL_DEPENDENCIES}) @@ -452,14 +464,9 @@ if (onnxruntime_USE_TENSORRT) ) source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_tensorrt_cc_srcs}) - add_library(onnxruntime_providers_tensorrt SHARED ${onnxruntime_providers_tensorrt_cc_srcs}) + onnxruntime_add_shared_library_module(onnxruntime_providers_tensorrt ${onnxruntime_providers_tensorrt_cc_srcs}) onnxruntime_add_include_to_target(onnxruntime_providers_tensorrt onnxruntime_common onnx flatbuffers) add_dependencies(onnxruntime_providers_tensorrt onnxruntime_providers_shared ${onnxruntime_EXTERNAL_DEPENDENCIES}) - if(WIN32) - target_link_directories(onnxruntime_providers_tensorrt PRIVATE ${onnxruntime_CUDA_HOME}/x64/lib64) - else() - target_link_directories(onnxruntime_providers_tensorrt PRIVATE ${onnxruntime_CUDA_HOME}/lib64) - endif() target_link_libraries(onnxruntime_providers_tensorrt PRIVATE ${onnxparser_link_libs} ${trt_link_libs} cudart onnxruntime_providers_shared protobuf::libprotobuf flatbuffers) target_include_directories(onnxruntime_providers_tensorrt PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) # ${CMAKE_CURRENT_BINARY_DIR} is so that #include "onnxruntime_config.h" inside tensor_shape.h is found @@ -614,7 +621,7 @@ if (onnxruntime_USE_OPENVINO) endif() source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_openvino_cc_srcs}) - add_library(onnxruntime_providers_openvino SHARED ${onnxruntime_providers_openvino_cc_srcs}) + onnxruntime_add_shared_library_module(onnxruntime_providers_openvino ${onnxruntime_providers_openvino_cc_srcs}) onnxruntime_add_include_to_target(onnxruntime_providers_openvino onnxruntime_common onnx) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/openvino DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers_openvino PROPERTIES LINKER_LANGUAGE CXX) diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index a211cb4431efb..9a3fdba3aad48 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -47,7 +47,7 @@ file(GLOB onnxruntime_pybind_srcs CONFIGURE_DEPENDS ${onnxruntime_pybind_srcs_pattern} ) -add_library(onnxruntime_pybind11_state MODULE ${onnxruntime_pybind_srcs}) +onnxruntime_add_shared_library_module(onnxruntime_pybind11_state ${onnxruntime_pybind_srcs}) if(MSVC) target_compile_options(onnxruntime_pybind11_state PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") endif() diff --git a/cmake/onnxruntime_training.cmake b/cmake/onnxruntime_training.cmake index 8776f9113270a..128b66d2fd875 100644 --- a/cmake/onnxruntime_training.cmake +++ b/cmake/onnxruntime_training.cmake @@ -88,7 +88,7 @@ file(GLOB_RECURSE training_mnist_src "${ORTTRAINING_SOURCE_DIR}/models/mnist/mnist_data_provider.cc" "${ORTTRAINING_SOURCE_DIR}/models/mnist/main.cc" ) -add_executable(onnxruntime_training_mnist ${training_mnist_src}) +onnxruntime_add_executable(onnxruntime_training_mnist ${training_mnist_src}) onnxruntime_add_include_to_target(onnxruntime_training_mnist onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers) target_include_directories(onnxruntime_training_mnist PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${eigen_INCLUDE_DIRS} ${CXXOPTS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx onnxruntime_training_runner) @@ -128,7 +128,7 @@ file(GLOB_RECURSE training_squeezene_src "${ORTTRAINING_SOURCE_DIR}/models/squeezenet/*.h" "${ORTTRAINING_SOURCE_DIR}/models/squeezenet/*.cc" ) -add_executable(onnxruntime_training_squeezenet ${training_squeezene_src}) +onnxruntime_add_executable(onnxruntime_training_squeezenet ${training_squeezene_src}) onnxruntime_add_include_to_target(onnxruntime_training_squeezenet onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers) target_include_directories(onnxruntime_training_squeezenet PUBLIC ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${eigen_INCLUDE_DIRS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx onnxruntime_training_runner) if(UNIX AND NOT APPLE) @@ -143,7 +143,7 @@ file(GLOB_RECURSE training_bert_src "${ORTTRAINING_SOURCE_DIR}/models/bert/*.h" "${ORTTRAINING_SOURCE_DIR}/models/bert/*.cc" ) -add_executable(onnxruntime_training_bert ${training_bert_src}) +onnxruntime_add_executable(onnxruntime_training_bert ${training_bert_src}) if(UNIX AND NOT APPLE) if (HAS_NO_MAYBE_UNINITIALIZED) @@ -162,7 +162,7 @@ file(GLOB_RECURSE training_pipeline_poc_src "${ORTTRAINING_SOURCE_DIR}/models/pipeline_poc/*.h" "${ORTTRAINING_SOURCE_DIR}/models/pipeline_poc/*.cc" ) -add_executable(onnxruntime_training_pipeline_poc ${training_pipeline_poc_src}) +onnxruntime_add_executable(onnxruntime_training_pipeline_poc ${training_pipeline_poc_src}) if(UNIX AND NOT APPLE) if (HAS_NO_MAYBE_UNINITIALIZED) @@ -181,7 +181,7 @@ file(GLOB_RECURSE training_gpt2_src "${ORTTRAINING_SOURCE_DIR}/models/gpt2/*.h" "${ORTTRAINING_SOURCE_DIR}/models/gpt2/*.cc" ) -add_executable(onnxruntime_training_gpt2 ${training_gpt2_src}) +onnxruntime_add_executable(onnxruntime_training_gpt2 ${training_gpt2_src}) if(UNIX AND NOT APPLE) if (HAS_NO_MAYBE_UNINITIALIZED) target_compile_options(onnxruntime_training_gpt2 PUBLIC "-Wno-maybe-uninitialized") diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 8678be2594474..e73397b2dc168 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -19,9 +19,9 @@ function(AddTest) list(REMOVE_DUPLICATES _UT_SOURCES) if (${CMAKE_SYSTEM_NAME} STREQUAL "iOS") - add_executable(${_UT_TARGET} ${TEST_SRC_DIR}/xctest/orttestmain.m) + onnxruntime_add_executable(${_UT_TARGET} ${TEST_SRC_DIR}/xctest/orttestmain.m) else() - add_executable(${_UT_TARGET} ${_UT_SOURCES}) + onnxruntime_add_executable(${_UT_TARGET} ${_UT_SOURCES}) endif() if (_UT_DEPENDS) @@ -718,7 +718,7 @@ if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS) list(APPEND onnx_test_libs onnxruntime_language_interop onnxruntime_pyop) endif() -add_executable(onnx_test_runner ${onnx_test_runner_src_dir}/main.cc) +onnxruntime_add_executable(onnx_test_runner ${onnx_test_runner_src_dir}/main.cc) if(MSVC) target_compile_options(onnx_test_runner PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") @@ -741,7 +741,7 @@ install(TARGETS onnx_test_runner if(onnxruntime_BUILD_BENCHMARKS) SET(BENCHMARK_DIR ${TEST_SRC_DIR}/onnx/microbenchmark) - add_executable(onnxruntime_benchmark + onnxruntime_add_executable(onnxruntime_benchmark ${BENCHMARK_DIR}/main.cc ${BENCHMARK_DIR}/modeltest.cc ${BENCHMARK_DIR}/pooling.cc @@ -798,7 +798,7 @@ endif() file(GLOB onnxruntime_perf_test_src CONFIGURE_DEPENDS ${onnxruntime_perf_test_src_patterns} ) -add_executable(onnxruntime_perf_test ${onnxruntime_perf_test_src} ${ONNXRUNTIME_ROOT}/core/platform/path_lib.cc) +onnxruntime_add_executable(onnxruntime_perf_test ${onnxruntime_perf_test_src} ${ONNXRUNTIME_ROOT}/core/platform/path_lib.cc) if(MSVC) target_compile_options(onnxruntime_perf_test PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") @@ -923,20 +923,20 @@ endif(onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS) #some ETW tools if(WIN32 AND onnxruntime_ENABLE_INSTRUMENT) - add_executable(generate_perf_report_from_etl ${ONNXRUNTIME_ROOT}/tool/etw/main.cc + onnxruntime_add_executable(generate_perf_report_from_etl ${ONNXRUNTIME_ROOT}/tool/etw/main.cc ${ONNXRUNTIME_ROOT}/tool/etw/eparser.h ${ONNXRUNTIME_ROOT}/tool/etw/eparser.cc ${ONNXRUNTIME_ROOT}/tool/etw/TraceSession.h ${ONNXRUNTIME_ROOT}/tool/etw/TraceSession.cc) target_compile_definitions(generate_perf_report_from_etl PRIVATE "_CONSOLE" "_UNICODE" "UNICODE") target_link_libraries(generate_perf_report_from_etl PRIVATE tdh Advapi32) - add_executable(compare_two_sessions ${ONNXRUNTIME_ROOT}/tool/etw/compare_two_sessions.cc + onnxruntime_add_executable(compare_two_sessions ${ONNXRUNTIME_ROOT}/tool/etw/compare_two_sessions.cc ${ONNXRUNTIME_ROOT}/tool/etw/eparser.h ${ONNXRUNTIME_ROOT}/tool/etw/eparser.cc ${ONNXRUNTIME_ROOT}/tool/etw/TraceSession.h ${ONNXRUNTIME_ROOT}/tool/etw/TraceSession.cc) target_compile_definitions(compare_two_sessions PRIVATE "_CONSOLE" "_UNICODE" "UNICODE") target_link_libraries(compare_two_sessions PRIVATE ${GETOPT_LIB_WIDE} tdh Advapi32) endif() -add_executable(onnxruntime_mlas_test ${TEST_SRC_DIR}/mlas/unittest.cpp) +onnxruntime_add_executable(onnxruntime_mlas_test ${TEST_SRC_DIR}/mlas/unittest.cpp) if(MSVC) target_compile_options(onnxruntime_mlas_test PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") diff --git a/cmake/onnxruntime_util.cmake b/cmake/onnxruntime_util.cmake index 54280348848cf..6b22ac9693957 100644 --- a/cmake/onnxruntime_util.cmake +++ b/cmake/onnxruntime_util.cmake @@ -11,7 +11,7 @@ file(GLOB_RECURSE onnxruntime_util_srcs CONFIGURE_DEPENDS source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_util_srcs}) add_library(onnxruntime_util ${onnxruntime_util_srcs}) -target_include_directories(onnxruntime_util PRIVATE ${ONNXRUNTIME_ROOT} ${MKLML_INCLUDE_DIR} PUBLIC ${eigen_INCLUDE_DIRS}) +target_include_directories(onnxruntime_util PRIVATE ${ONNXRUNTIME_ROOT} PUBLIC ${eigen_INCLUDE_DIRS}) if (onnxruntime_USE_CUDA) target_include_directories(onnxruntime_util PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) endif() diff --git a/cmake/winml_unittests.cmake b/cmake/winml_unittests.cmake index 70198bf1190cc..3a2620c65efc1 100644 --- a/cmake/winml_unittests.cmake +++ b/cmake/winml_unittests.cmake @@ -43,7 +43,7 @@ function(add_winml_test) list(REMOVE_DUPLICATES _UT_DEPENDS) endif() - add_executable(${_UT_TARGET} ${_UT_SOURCES}) + onnxruntime_add_executable(${_UT_TARGET} ${_UT_SOURCES}) onnxruntime_add_include_to_target(${_UT_TARGET} onnx_proto) source_group(TREE ${WINML_TEST_SRC_DIR} FILES ${_UT_SOURCES}) set_winml_target_properties(${_UT_TARGET}) diff --git a/orttraining/orttraining/training_ops/cuda/activation/bias_gelu_grad_impl.cu b/orttraining/orttraining/training_ops/cuda/activation/bias_gelu_grad_impl.cu index f42a2360aae63..2007036db7815 100644 --- a/orttraining/orttraining/training_ops/cuda/activation/bias_gelu_grad_impl.cu +++ b/orttraining/orttraining/training_ops/cuda/activation/bias_gelu_grad_impl.cu @@ -62,8 +62,8 @@ void LaunchBiasGeluGradDxKernel( // each grid row handles bias_size elements // there are input_size / bias_size rows constexpr int num_elements_per_thread = GridDim::maxElementsPerThread; - const auto num_threads_per_block = - std::min(CeilDiv(bias_size, num_elements_per_thread), static_cast(GridDim::maxThreadsPerBlock)); + const int num_threads_per_block = + std::min(static_cast(CeilDiv(bias_size, num_elements_per_thread)), static_cast(GridDim::maxThreadsPerBlock)); const auto grid_width = CeilDiv(bias_size, num_elements_per_thread * num_threads_per_block); const auto grid_height = input_size / bias_size; diff --git a/orttraining/orttraining/training_ops/cuda/math/mixed_precision_scale.cu b/orttraining/orttraining/training_ops/cuda/math/mixed_precision_scale.cu index b557e28a91d65..a4c46b12aa3cc 100644 --- a/orttraining/orttraining/training_ops/cuda/math/mixed_precision_scale.cu +++ b/orttraining/orttraining/training_ops/cuda/math/mixed_precision_scale.cu @@ -1,3 +1,10 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + #include #include "core/providers/cuda/cu_inc/common.cuh" #include "mixed_precision_scale.h" @@ -21,7 +28,7 @@ void Impl_MixedPrecisionScale( const float* scale_data, DstT* output_data, size_t count){ - int blocksPerGrid = CeilDiv(count, GridDim::maxThreadsPerBlock); + int blocksPerGrid = static_cast(CeilDiv(count, GridDim::maxThreadsPerBlock)); CUDA_LONG N = static_cast(count); _MixedPrecisionScale<<>>( input_data, diff --git a/orttraining/orttraining/training_ops/cuda/nn/layer_norm_impl.cu b/orttraining/orttraining/training_ops/cuda/nn/layer_norm_impl.cu index cb9be8d6fcfff..00bdc2d5253b2 100644 --- a/orttraining/orttraining/training_ops/cuda/nn/layer_norm_impl.cu +++ b/orttraining/orttraining/training_ops/cuda/nn/layer_norm_impl.cu @@ -21,7 +21,9 @@ // /* Modifications Copyright (c) Microsoft. */ - +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif #include "orttraining/training_ops/cuda/nn/layer_norm_impl.h" #include "core/providers/cuda/cu_inc/common.cuh" @@ -500,7 +502,7 @@ void HostLayerNormGradient( grad_beta); // compute grad_input const uint64_t maxGridY = prop.maxGridSize[1]; - const dim3 blocks1(1, std::min((uint64_t)n1, maxGridY), 1); + const dim3 blocks1(1, std::min(static_cast(n1), static_cast(maxGridY)), 1); const dim3 threads1(warp_size, 4, 1); int nshared = threads1.y > 1 ? threads1.y * threads1.x * sizeof(U) : 0; diff --git a/orttraining/orttraining/training_ops/cuda/reduction/all.cu b/orttraining/orttraining/training_ops/cuda/reduction/all.cu index adae90c99d50c..64c62523d5d75 100644 --- a/orttraining/orttraining/training_ops/cuda/reduction/all.cu +++ b/orttraining/orttraining/training_ops/cuda/reduction/all.cu @@ -6,7 +6,9 @@ #include #include #include - +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif namespace onnxruntime { namespace cuda { diff --git a/orttraining/orttraining/training_ops/cuda/tensor/gather_grad_impl.cu b/orttraining/orttraining/training_ops/cuda/tensor/gather_grad_impl.cu index 9ea6196973e4f..60713126aec9f 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/gather_grad_impl.cu +++ b/orttraining/orttraining/training_ops/cuda/tensor/gather_grad_impl.cu @@ -1,5 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif #include "orttraining/training_ops/cuda/tensor/gather_grad_impl.h" diff --git a/orttraining/orttraining/training_ops/cuda/tensor/gather_nd_grad_impl.cu b/orttraining/orttraining/training_ops/cuda/tensor/gather_nd_grad_impl.cu index 46082ed41cbda..d733887af9485 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/gather_nd_grad_impl.cu +++ b/orttraining/orttraining/training_ops/cuda/tensor/gather_nd_grad_impl.cu @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. - +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif #include "orttraining/training_ops/cuda/tensor/gather_nd_grad_impl.h" #include "core/providers/cuda/cu_inc/common.cuh" @@ -29,7 +31,7 @@ void GatherNDGradImpl( void* output_data, const size_t slice_size, const int64_t* input_slice_offsets_data) { - const auto blocks_per_grid = CeilDiv(num_slices * slice_size, GridDim::maxThreadsPerBlock); + const unsigned int blocks_per_grid = static_cast(CeilDiv(num_slices * slice_size, GridDim::maxThreadsPerBlock)); _GatherNDGradKernel<<>>( num_slices, static_cast(update_data), static_cast(output_data), slice_size, input_slice_offsets_data); } diff --git a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml index 2b2a5f9012819..16e44b3d1fbc6 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml @@ -1,49 +1,47 @@ -jobs: -- job: Linux_C_API_Packaging_CPU_x64 - workspace: - clean: all - timeoutInMinutes: 60 - pool: - vmImage: 'ubuntu-latest' - steps: - - template: templates/set-version-number-variables-step.yml - - template: templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecentoscpubuild - - task: CmdLine@2 - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecentoscpubuild python3 \ - /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ - --skip_submodule_sync --parallel --build_shared_lib --use_openmp - workingDirectory: $(Build.SourcesDirectory) - - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml - parameters: - buildConfig: 'Release' - artifactName: 'onnxruntime-linux-x64-$(OnnxRuntimeVersion)' - libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' - commitId: $(OnnxRuntimeGitCommitHash) +parameters: +- name: RunOnnxRuntimeTests + displayName: Run Tests? + type: boolean + default: true + +- name: DoCompliance + displayName: Run Compliance Tasks? + type: boolean + default: true + +- name: DoEsrp + displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release. + type: boolean + default: false + +- name: IsReleaseBuild + displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release. + type: boolean + default: false - - template: templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - template: templates/clean-agent-build-directory-step.yml +jobs: +- template: templates/c-api-cpu.yml + parameters: + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime' + AdditionalBuildFlags: '--use_openmp' + AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos' - job: Linux_C_API_Packaging_GPU_x64 workspace: clean: all timeoutInMinutes: 120 pool: 'Linux-GPU-CUDA10' + variables: + CUDA_VERSION: '11.0' steps: - template: templates/set-version-number-variables-step.yml - template: templates/get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda10_2 + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11 Context: tools/ci_build/github/linux/docker DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" Repository: onnxruntimegpubuild @@ -54,231 +52,35 @@ jobs: docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimegpubuild \ /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ - --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 + --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home=/usr/local/cuda-$(CUDA_VERSION) --cudnn_home=/usr/local/cuda-$(CUDA_VERSION) workingDirectory: $(Build.SourcesDirectory) + - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml parameters: buildConfig: 'Release' artifactName: 'onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion)' + artifactNameNoVersionString: 'onnxruntime-linux-x64-gpu' libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' commitId: $(OnnxRuntimeGitCommitHash) - - template: templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - template: templates/clean-agent-build-directory-step.yml - - -- job: MacOS_C_API_Packaging_CPU_x64 - workspace: - clean: all - pool: - vmImage: 'macOS-10.14' - timeoutInMinutes: 120 - steps: - - template: templates/set-version-number-variables-step.yml - - script: | - set -e - pushd . - cd $(Build.SourcesDirectory)/cmake/external/protobuf - cmake ./cmake -DCMAKE_INSTALL_PREFIX=$(Build.BinariesDirectory)/protobuf -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo - make -j$(getconf _NPROCESSORS_ONLN) - make install - popd - export PATH=$(Build.BinariesDirectory)/protobuf/bin:$PATH - export ONNX_ML=1 - export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" - sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer - brew install libomp - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release - displayName: 'Build and Test MacOS' - - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml - parameters: - buildConfig: 'Release' - artifactName: 'onnxruntime-osx-x64-$(OnnxRuntimeVersion)' - libraryName: 'libonnxruntime.$(OnnxRuntimeVersion).dylib' - commitId: $(OnnxRuntimeGitCommitHash) - - template: templates/component-governance-component-detection-steps.yml parameters : condition : 'succeeded' - template: templates/clean-agent-build-directory-step.yml -- job: Windows_Packaging_CPU - workspace: - clean: all - pool: 'Win-CPU-2019' - timeoutInMinutes: 160 - strategy: - maxParallel: 2 - matrix: - x86: - EnvSetupScript: setup_env_x86.bat - buildArch: x86 - msbuildArch: x86 - msbuildPlatform: Win32 - buildparameter: --x86 - x64: - EnvSetupScript: setup_env.bat - buildArch: x64 - msbuildArch: amd64 - msbuildPlatform: x64 - buildparameter: - steps: - - template: templates/telemetry-steps.yml - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: $(buildArch) - - - task: BatchScript@1 - displayName: 'setup env' - inputs: - filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' - modifyEnvironment: true - workingFolder: '$(Build.BinariesDirectory)' - - - script: | - python -m pip install -q pyopenssl setuptools wheel numpy - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Install python modules' - - powershell: | - $Env:USE_MSVC_STATIC_RUNTIME=1 - $Env:ONNX_ML=1 - $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static" - python setup.py bdist_wheel - python -m pip uninstall -y onnx - Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} - workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx' - displayName: 'Install ONNX' - - - template: templates/set-test-data-variables-step.yml - - template: templates/set-version-number-variables-step.yml - - - - task: PythonScript@0 - displayName: 'Generate cmake config' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --use_openmp --build_shared_lib --enable_onnx_tests $(TelemetryOption) $(buildparameter)' - workingDirectory: '$(Build.BinariesDirectory)' - - - task: VSBuild@1 - displayName: 'Build' - inputs: - solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' - platform: $(msbuildPlatform) - configuration: RelWithDebInfo - msbuildArchitecture: $(buildArch) - maximumCpuCount: true - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - createLogFile: true - - - task: PythonScript@0 - displayName: 'test' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --use_openmp --build_shared_lib --enable_onnx_tests $(TelemetryOption) $(buildparameter)' - workingDirectory: '$(Build.BinariesDirectory)' - - - template: templates/c-api-artifacts-package-and-publish-steps-windows.yml - parameters: - buildConfig: RelWithDebInfo - artifactName: 'onnxruntime-win-$(buildArch)-$(OnnxRuntimeVersion)' - commitId: $(OnnxRuntimeGitCommitHash) - - template: templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() - -- job: Windows_Packaging_GPU - workspace: - clean: all - pool: 'Win-GPU-2019' - timeoutInMinutes: 120 - variables: - EnvSetupScript: setup_env_cuda.bat +- template: templates/win-ci.yml + parameters: + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + ort_build_pool_name: 'Win-GPU-2019' + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime.Gpu' + job_name_suffix: gpu + EnvSetupScript: setup_env_cuda_11.bat buildArch: x64 - msbuildArch: amd64 msbuildPlatform: x64 - CUDA_VERSION: '10.2' - buildparameter: --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows10-x64-v8.0.3.33\cuda" - steps: - - template: templates/telemetry-steps.yml - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: $(buildArch) - - - task: BatchScript@1 - displayName: 'setup env' - inputs: - filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' - modifyEnvironment: true - workingFolder: '$(Build.BinariesDirectory)' - - - script: | - python -m pip install -q pyopenssl setuptools wheel numpy - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Install python modules' - - powershell: | - $Env:USE_MSVC_STATIC_RUNTIME=1 - $Env:ONNX_ML=1 - $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static" - python setup.py bdist_wheel - python -m pip uninstall -y onnx - Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} - workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx' - displayName: 'Install ONNX' - - - template: templates/set-test-data-variables-step.yml - - template: templates/set-version-number-variables-step.yml - - - task: PythonScript@0 - displayName: 'Generate cmake config' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) $(buildparameter)' - workingDirectory: '$(Build.BinariesDirectory)' - - - - task: VSBuild@1 - displayName: 'Build' - inputs: - solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' - platform: $(msbuildPlatform) - configuration: RelWithDebInfo - msbuildArchitecture: $(buildArch) - maximumCpuCount: true - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - createLogFile: true - - - task: PythonScript@0 - displayName: 'test' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) $(buildparameter)' - workingDirectory: '$(Build.BinariesDirectory)' - - - template: templates/c-api-artifacts-package-and-publish-steps-windows.yml - parameters: - buildConfig: RelWithDebInfo - artifactName: 'onnxruntime-win-$(buildArch)-gpu-$(OnnxRuntimeVersion)' - commitId: $(OnnxRuntimeGitCommitHash) - - template: templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() + packageName: x64-gpu + buildparameter: --use_cuda --cuda_version=11.0 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0" --cudnn_home="C:\local\cudnn-11.0-windows-x64-v8.0.2.39\cuda" --enable_onnx_tests --enable_wcos + runTests: true + buildJava: false diff --git a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml deleted file mode 100644 index b13cbe3ba399b..0000000000000 --- a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml +++ /dev/null @@ -1,361 +0,0 @@ -schedules: -- cron: "0 8 * * *" - displayName: Daily Build - branches: - include: - - master - always: true - -jobs: -- job: Linux_Java_API_Build_CPU_x64 - workspace: - clean: all - timeoutInMinutes: 60 - pool: 'Linux-CPU' - steps: - - template: templates/set-version-number-variables-step.yml - - template: templates/linux-set-variables-and-download.yml - - - template: templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecentoscpubuild - - - task: CmdLine@2 - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecentoscpubuild /bin/bash -c "python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --build_java --use_openmp --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" - workingDirectory: $(Build.SourcesDirectory) - displayName: 'Run build and test' - - - template: templates/java-api-artifacts-package-and-publish-steps-posix.yml - parameters: - arch: 'linux-x64' - buildConfig: 'Release' - artifactName: 'onnxruntime-java-linux-x64' - version: '$(OnnxRuntimeVersion)' - libraryName: 'libonnxruntime.so' - nativeLibraryName: 'libonnxruntime4j_jni.so' - - - template: templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - - template: templates/clean-agent-build-directory-step.yml - -- job: MacOS_Java_API_Build_CPU_x64 - workspace: - clean: all - pool: - vmImage: 'macOS-10.14' - steps: - - template: templates/set-version-number-variables-step.yml - - - script: | - export JAVA_HOME=$(java_home -v 11) - sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_java --build_shared_lib --use_openmp --config RelWithDebInfo - - displayName: 'Build and Test MacOS' - - template: templates/java-api-artifacts-package-and-publish-steps-posix.yml - parameters: - arch: 'osx-x64' - buildConfig: 'RelWithDebInfo' - artifactName: 'onnxruntime-java-osx-x64' - version: '$(OnnxRuntimeVersion)' - libraryName: 'libonnxruntime.dylib' - nativeLibraryName: 'libonnxruntime4j_jni.dylib' - - - template: templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - - template: templates/clean-agent-build-directory-step.yml - -- job: Windows_Java_API_Build_CPU_x64 - workspace: - clean: all - pool: 'Win-CPU-2019' - timeoutInMinutes: 160 - strategy: - maxParallel: 2 - matrix: -# x86: - #EnvSetupScript: setup_env_x86.bat - #buildArch: x86 - #msbuildArch: x86 - #msbuildPlatform: Win32 - #buildparameter: --x86 - x64: - EnvSetupScript: setup_env.bat - buildArch: x64 - msbuildArch: amd64 - msbuildPlatform: x64 - buildparameter: - - steps: - - template: templates/telemetry-steps.yml - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: $(buildArch) - - - task: BatchScript@1 - displayName: 'setup env' - inputs: - filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' - modifyEnvironment: true - workingFolder: '$(Build.BinariesDirectory)' - - - script: | - python -m pip install -q pyopenssl setuptools wheel numpy - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Install python modules' - - - powershell: | - $Env:USE_MSVC_STATIC_RUNTIME=1 - $Env:ONNX_ML=1 - $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static" - python setup.py bdist_wheel - python -m pip uninstall -y onnx - Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} - workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx' - displayName: 'Install ONNX' - - - template: templates/set-test-data-variables-step.yml - - template: templates/set-version-number-variables-step.yml - - task: PythonScript@0 - displayName: 'Generate cmake config' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --enable_wcos --build_java --use_telemetry --update --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' - workingDirectory: '$(Build.BinariesDirectory)' - - - - task: VSBuild@1 - displayName: 'Build' - inputs: - solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' - platform: $(msbuildPlatform) - configuration: RelWithDebInfo - msbuildArchitecture: $(buildArch) - maximumCpuCount: true - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - createLogFile: true - - - - task: PythonScript@0 - displayName: 'test' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_java --enable_wcos --build_java --test --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' - workingDirectory: '$(Build.BinariesDirectory)' - - - template: templates/java-api-artifacts-package-and-publish-steps-windows.yml - parameters: - buildConfig: RelWithDebInfo - artifactName: 'onnxruntime-java-win-$(buildArch)' - version: '$(OnnxRuntimeVersion)' - commitId: $(OnnxRuntimeGitCommitHash) - - - template: templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - - task: Semmle@0 - inputs: - sourceCodeDirectory: '$(Build.SourcesDirectory)\java' - language: 'java' - cleanupBuildCommands: 'C:\ProgramData\chocolatey\bin\gradle.exe -Dorg.gradle.daemon=false clean' - buildCommands: 'C:\ProgramData\chocolatey\bin\gradle.exe -Dorg.gradle.daemon=false jar' - querySuite: 'Recommended' - timeout: '7200' - ram: '16384' - addProjectDirToScanningExclusionList: true - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 - displayName: 'Publish Security Analysis Logs' - continueOnError: true - - - template: templates/clean-agent-build-directory-step.yml - -- job: Jar_Packaging - workspace: - clean: all - pool: 'Win-CPU-2019' - dependsOn: - - Windows_Java_API_Build_CPU_x64 - - Linux_Java_API_Build_CPU_x64 - - MacOS_Java_API_Build_CPU_x64 - condition: succeeded() - steps: - - template: templates/set-version-number-variables-step.yml - - - task: DownloadPipelineArtifact@2 - displayName: 'Download Pipeline Artifact - Win x64' - inputs: - buildType: 'current' - artifactName: 'drop-onnxruntime-java-win-x64' - targetPath: '$(Build.BinariesDirectory)\java-artifact' - - - task: DownloadPipelineArtifact@2 - displayName: 'Download Pipeline Artifact - Linux x64' - inputs: - buildType: 'current' - artifactName: 'drop-onnxruntime-java-linux-x64' - targetPath: '$(Build.BinariesDirectory)\java-artifact' - - - task: DownloadPipelineArtifact@2 - displayName: 'Download Pipeline Artifact - MacOS x64' - inputs: - buildType: 'current' - artifactName: 'drop-onnxruntime-java-osx-x64' - targetPath: '$(Build.BinariesDirectory)\java-artifact' - - - task: ExtractFiles@1 - inputs: - archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64.zip' - destinationFolder: '$(Build.BinariesDirectory)\java-artifact' - cleanDestinationFolder: false - - - task: ExtractFiles@1 - inputs: - archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64.zip' - destinationFolder: '$(Build.BinariesDirectory)\java-artifact' - cleanDestinationFolder: false - - - task: ExtractFiles@1 - inputs: - archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x64.zip' - destinationFolder: '$(Build.BinariesDirectory)\java-artifact' - cleanDestinationFolder: false - - - task: CmdLine@2 - inputs: - script: | - @echo on - pushd onnxruntime-java-linux-x64 - jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64\testing.jar libcustom_op_library.so - del /F /Q libcustom_op_library.so - jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64\onnxruntime-$(OnnxRuntimeVersion).jar . - popd - pushd onnxruntime-java-osx-x64 - jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64\testing.jar libcustom_op_library.dylib - del /F /Q libcustom_op_library.dylib - jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64\onnxruntime-$(OnnxRuntimeVersion).jar . - popd - workingDirectory: '$(Build.BinariesDirectory)\java-artifact' - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64' - artifact: 'onnxruntime-java' - publishLocation: 'pipeline' - - - template: templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - -- job: Final_Jar_Testing_Windows - workspace: - clean: all - pool: 'Win-CPU-2019' - timeoutInMinutes: 60 - dependsOn: - Jar_Packaging - steps: - - template: templates/set-version-number-variables-step.yml - - - task: DownloadPipelineArtifact@2 - displayName: 'Download Final Jar' - inputs: - buildType: 'current' - artifactName: 'onnxruntime-java' - targetPath: '$(Build.BinariesDirectory)\final-jar' - - - task: CmdLine@2 - inputs: - script: | - mkdir test - pushd test - jar xf $(Build.BinariesDirectory)\final-jar\testing.jar - popd - powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar" - powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -OutFile protobuf-java-3.9.2.jar" - java -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.9.2.jar;onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner - workingDirectory: '$(Build.BinariesDirectory)\final-jar' - -- job: Final_Jar_Testing_Linux - workspace: - clean: all - pool: 'Linux-CPU' - timeoutInMinutes: 60 - dependsOn: - Jar_Packaging - steps: - - template: templates/set-version-number-variables-step.yml - - task: DownloadPipelineArtifact@2 - displayName: 'Download Final Jar' - inputs: - buildType: 'current' - artifactName: 'onnxruntime-java' - targetPath: '$(Build.BinariesDirectory)/final-jar' - - - task: CmdLine@2 - inputs: - script: | - sudo apt-get install -y openjdk-8-jdk - sudo apt autoremove - PATH=/usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin:${PATH} - echo "Java Version" - java --version - mkdir test - pushd test - jar xf $(Build.BinariesDirectory)/final-jar/testing.jar - popd - wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./ - wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./ - LD_LIBRARY_PATH=./test:${LD_LIBRARY_PATH} - java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.9.2.jar:./onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner - workingDirectory: '$(Build.BinariesDirectory)/final-jar' - -- job: Final_Jar_Testing_MacOs - workspace: - clean: all - pool: - vmImage: 'macOS-10.14' - timeoutInMinutes: 60 - dependsOn: - Jar_Packaging - steps: - - template: templates/set-version-number-variables-step.yml - - - task: DownloadPipelineArtifact@2 - displayName: 'Download Final Jar' - inputs: - buildType: 'current' - artifactName: 'onnxruntime-java' - targetPath: '$(Build.BinariesDirectory)/final-jar' - - - task: CmdLine@2 - inputs: - script: | - echo "Java Version" - java --version - mkdir test - pushd test - jar xf $(Build.BinariesDirectory)/final-jar/testing.jar - popd - wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./ - wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./ - DYLD_LIBRARY_PATH=./test:${DYLD_LIBRARY_PATH} - java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.9.2.jar:./onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner - workingDirectory: '$(Build.BinariesDirectory)/final-jar' - diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-featruizers-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-featruizers-pipeline.yml deleted file mode 100644 index 870b232a03a14..0000000000000 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-featruizers-pipeline.yml +++ /dev/null @@ -1,17 +0,0 @@ -schedules: -- cron: "0 3 * * *" - displayName: Featurizers Nightly Build - branches: - include: - - master - always: true - -variables: - PackageName: 'Microsoft.ML.ORTFeaturizers' - -jobs: -- template: templates/cpu-featurizers.yml - parameters: - AgentPool : 'Win-CPU-2019' - DoEsrp: 'true' - DoCompliance: 'true' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-noopenmp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-noopenmp-pipeline.yml index 13b0eec0f5b73..8e6c181865524 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-noopenmp-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-noopenmp-pipeline.yml @@ -1,17 +1,35 @@ -schedules: -- cron: "0 8 * * *" - displayName: Daily Build - branches: - include: - - master - always: true +parameters: +- name: RunOnnxRuntimeTests + displayName: Run Tests? + type: boolean + default: true + +- name: DoCompliance + displayName: Run Compliance Tasks? + type: boolean + default: true + +- name: DoEsrp + displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release. + type: boolean + default: false + +- name: IsReleaseBuild + displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release. + type: boolean + default: false variables: PackageName: 'Microsoft.ML.OnnxRuntime.NoOpenMP' -jobs: -- template: templates/cpu-noopenmp.yml +jobs: +- template: ../templates/c-api-cpu.yml parameters: - AgentPool : 'Win-CPU-2019' - DoEsrp: 'true' - DoCompliance: 'true' + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime.NoOpenMP' + AdditionalBuildFlags: '' + AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos' + BuildVariant: 'default' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml deleted file mode 100644 index 3eaaf991398c6..0000000000000 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml +++ /dev/null @@ -1,14 +0,0 @@ -schedules: -- cron: "0 8 * * *" - displayName: Daily Build - branches: - include: - - master - always: true - -jobs: -- template: templates/cpu.yml - parameters: - AgentPool : 'Win-CPU-2019' - DoEsrp: 'true' - DoCompliance: 'true' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-esrp-pipeline.yml index bbbf066792cd1..3049ac43893d7 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-esrp-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-esrp-pipeline.yml @@ -1,17 +1,37 @@ -schedules: -- cron: "0 14 * * *" - displayName: Daily Build - branches: - include: - - master - always: true +parameters: +- name: RunOnnxRuntimeTests + displayName: Run Tests? + type: boolean + default: true + +- name: DoCompliance + displayName: Run Compliance Tasks? + type: boolean + default: true + +- name: DoEsrp + displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release. + type: boolean + default: false + +- name: IsReleaseBuild + displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release. + type: boolean + default: false variables: DisableContribOps: 'ON' DisableMlOps: 'ON' jobs: -- template: templates/cpu-nocontribops-arm64.yml +- template: ../templates/c-api-cpu-no-java.yml parameters: - AgentPool : 'Win-CPU-2019' - DoEsrp: 'true' + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime' + AdditionalBuildFlags: '--disable_contrib_ops --disable_ml_ops' + AdditionalWinBuildFlags: '--enable_msvc_static_runtime' + BuildVariant: 'default' + PublishingNuget: false \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-pipeline.yml deleted file mode 100644 index 5e0ca516fc73e..0000000000000 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-pipeline.yml +++ /dev/null @@ -1,9 +0,0 @@ -variables: - DisableContribOps: 'ON' - DisableMlOps: 'ON' - -jobs: -- template: templates/cpu-nocontribops-arm64.yml - parameters: - AgentPool : 'Win-CPU-2019' - DoEsrp: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-pipeline.yml deleted file mode 100644 index 261fa655cb296..0000000000000 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-pipeline.yml +++ /dev/null @@ -1,5 +0,0 @@ -jobs: -- template: templates/cpu.yml - parameters: - AgentPool : 'Win-CPU-2019' - DoEsrp: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml deleted file mode 100644 index 28b8e20894296..0000000000000 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml +++ /dev/null @@ -1,205 +0,0 @@ -parameters: - DoEsrp: 'false' - DoCompliance: 'false' - PackageName: 'Microsoft.ML.ORTFeaturizers' - -jobs: -- template: ../../templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - ArtifactName: 'drop-nuget' - JobName: 'Windows_CI_Dev' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --enable_wcos --use_winml --cmake_generator "Visual Studio 16 2019"' - BuildArch: 'x64' - EnvSetupScript: 'setup_env.bat' - sln_platform: 'x64' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - OrtPackageId: 'Microsoft.ML.ORTFeaturizers' - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:OrtPackageId=Microsoft.ML.ORTFeaturizers - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- template: ../../templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - ArtifactName: 'drop-win-x86-zip' - JobName: 'Windows_CI_Dev_x86' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --enable_wcos --x86 --use_winml --cmake_generator "Visual Studio 16 2019"' - BuildArch: 'x86' - EnvSetupScript: 'setup_env_x86.bat' - sln_platform: 'Win32' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - OrtPackageId: 'Microsoft.ML.ORTFeaturizers' - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage /p:OrtPackageId=Microsoft.ML.ORTFeaturizers - cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\ - ren *.nupkg win-x86.zip - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- job: 'Linux_CI_Dev' - workspace: - clean: all - pool: $(AgentPoolLinux) - steps: - - template: ../../templates/set-version-number-variables-step.yml - - template: ../../templates/linux-set-variables-and-download.yml - - task: CmdLine@2 - inputs: - script: | - docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . - workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - - task: CmdLine@2 - inputs: - script: | - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --use_featurizers --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" - workingDirectory: $(Build.SourcesDirectory) - - script: | - set -e -x - mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 - cd $(Build.BinariesDirectory)/linux-x64 - zip -r linux-x64.zip linux-x64 - cp $(Build.BinariesDirectory)/linux-x64/linux*.zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata - ls -al $(Build.ArtifactStagingDirectory) - displayName: 'Create Artifacts' - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline Artifact' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.ArtifactStagingDirectory)' - - template: ../../templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - template: ../../templates/clean-agent-build-directory-step.yml - -- template: ../../templates/mac-ci.yml - parameters: - AgentPool : $(AgentPoolMacOS) - JobName: 'MacOS_CI_Dev' - BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --use_featurizers --build_shared_lib --use_openmp --config RelWithDebInfo' - DoNugetPack : 'true' - NuPackScript: | - set -e -x - mkdir $(Build.BinariesDirectory)/osx-x64 - find $(Build.BinariesDirectory) - cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ - dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM - strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib - find $(Build.BinariesDirectory)/osx-x64 -ls - cwd=`pwd` - cd $(Build.BinariesDirectory) - zip -r osx-x64.zip osx-x64 - cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/RelWithDebInfo/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata - cd $cwd - -- job: NuGet_Packaging - workspace: - clean: all - pool: 'Win-CPU-2019' - dependsOn: - - Windows_CI_Dev - - Windows_CI_Dev_x86 - - Linux_CI_Dev - - MacOS_CI_Dev - condition: succeeded() - steps: - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - NuGet' - inputs: - artifactName: 'drop-nuget' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Win-x86' - inputs: - artifactName: 'drop-win-x86-zip' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Linux' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - MacOS' - inputs: - artifactName: 'drop-osx' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - template: bundle_dlls.yml - - - template: ../../templates/esrp_nuget.yml - parameters: - DisplayName: 'ESRP - sign NuGet package' - FolderPath: '$(Build.ArtifactStagingDirectory)' - DoEsrp: ${{ parameters.DoEsrp }} - - - template: ../../templates/validate-nuget.yml - parameters: - NugetPath: '$(Build.ArtifactStagingDirectory)' - NugetPackage: 'Microsoft.ML.ORTFeaturizers*nupkg' - PlatformsSupported: 'win-x64,win-x86,linux-x64,osx-x64' - VerifyNugetSigning: ${{ parameters.DoEsrp }} - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline NuGet Artifact' - inputs: - artifactName: 'drop-signed-nuget' - targetPath: '$(Build.ArtifactStagingDirectory)' - -- template: test_all_os.yml - -- job: Publish_NuGet_Package_And_Report - workspace: - clean: all - pool: - name: Hosted Windows 2019 with VS2019 - # AzureFileCopy@3 task has some bug that it depends on a particular version of azure power shell, - # which is not available in OnnxRuntime build VMs, but available in the latest hosted agents. - # So, all the copy/publish jobs are being run on hosted agent - # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy - demands: azureps - condition: and (succeeded(), and (${{ parameters.DoEsrp }}, eq(variables['Build.SourceBranch'], 'refs/heads/master'))) - dependsOn: - - NuGet_Test_Win - - NuGet_Test_Linux - - NuGet_Test_MacOS - steps: - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Signed NuGet Package' - inputs: - artifactName: 'drop-signed-nuget' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - - - task: NuGetCommand@2 - displayName: 'Copy Signed Native NuGet Package to ORT-NIGHTLY' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - command: 'push' - packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.ORTFeaturizers.$(NuGetPackageVersionNumber).nupkg' - publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7' - - - task: NuGetCommand@2 - displayName: 'Copy Signed Managed NuGet Package to ORT-NIGHTLY' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - command: 'push' - packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg' - publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7' - diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml deleted file mode 100644 index 4e79eeca06d74..0000000000000 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml +++ /dev/null @@ -1,240 +0,0 @@ -parameters: - DoEsrp: 'false' - -jobs: -- template: ../../templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - ArtifactName: 'drop-nuget' - JobName: 'Windows_CI_Dev' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --cmake_generator "Visual Studio 16 2019" --disable_contrib_ops --disable_ml_ops --enable_msvc_static_runtime' - BuildArch: 'x64' - EnvSetupScript: 'setup_env.bat' - sln_platform: 'x64' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- template: ../../templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - ArtifactName: 'drop-win-x86-zip' - JobName: 'Windows_CI_Dev_x86' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --x86 --cmake_generator "Visual Studio 16 2019" --disable_contrib_ops --disable_ml_ops --enable_msvc_static_runtime' - BuildArch: 'x86' - EnvSetupScript: 'setup_env_x86.bat' - sln_platform: 'Win32' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: 'false' - DoEsrp: ${{ parameters.DoEsrp }} - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage - cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\ - ren *.nupkg win-x86.zip - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- template: ../../templates/win-ci-arm.yml - parameters: - AgentPool : 'Win-CPU-2019' - JobName: 'Windows_Arm64_Dev' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_generator "Visual Studio 16 2019" --disable_contrib_ops --disable_ml_ops --enable_msvc_static_runtime --arm64' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: 'false' - MsbuildArguments: '/m /p:platform=arm64' - DoEsrp: ${{ parameters.DoEsrp }} - ArtifactName: 'drop-nuget-arm64' - NuPackScript: | - mkdir $(Build.BinariesDirectory)\arm64\runtimes\win10-arm\native - cd $(Build.BinariesDirectory)\arm64 - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime.pdb $(Build.BinariesDirectory)\arm64\runtimes\win10-arm\native - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime.lib $(Build.BinariesDirectory)\arm64\runtimes\win10-arm\native - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime.dll $(Build.BinariesDirectory)\arm64\runtimes\win10-arm\native - powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe" - zip -r win10-arm.zip runtimes - copy *.zip $(Build.ArtifactStagingDirectory) - -- job: 'Linux_CI_Dev' - workspace: - clean: all - timeoutInMinutes: 120 - pool: $(AgentPoolLinux) - steps: - - template: ../../templates/set-version-number-variables-step.yml - - template: ../../templates/linux-set-variables-and-download.yml - - template: ../../templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecentoscpubuild - - task: CmdLine@2 - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecentoscpubuild /bin/bash -c "python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --enable_onnx_tests --disable_contrib_ops --disable_ml_ops && cd /build/Release && make install DESTDIR=/build/linux-x64" - workingDirectory: $(Build.SourcesDirectory) - - script: | - set -e -x - mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 - cd $(Build.BinariesDirectory)/linux-x64 - zip -r linux-x64.zip linux-x64 - cp $(Build.BinariesDirectory)/linux-x64/linux*.zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata - ls -al $(Build.ArtifactStagingDirectory) - displayName: 'Create Artifacts' - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline Artifact' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.ArtifactStagingDirectory)' - - template: ../../templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - template: ../../templates/clean-agent-build-directory-step.yml - -- template: ../../templates/mac-ci.yml - parameters: - AgentPool : $(AgentPoolMacOS) - JobName: 'MacOS_CI_Dev' - BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --disable_contrib_ops --disable_ml_ops --config RelWithDebInfo' - DoNugetPack : 'true' - NuPackScript: | - set -e -x - mkdir $(Build.BinariesDirectory)/osx-x64 - find $(Build.BinariesDirectory) - cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ - dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM - strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib - find $(Build.BinariesDirectory)/osx-x64 - cwd=`pwd` - cd $(Build.BinariesDirectory) - zip -r osx-x64.zip osx-x64 - cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/RelWithDebInfo/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata - cd $cwd - -- job: NuGet_Packaging - workspace: - clean: all - pool: 'Win-CPU-2019' - dependsOn: - - Windows_CI_Dev - - Windows_CI_Dev_x86 - - Windows_Arm64_Dev - - Linux_CI_Dev - - MacOS_CI_Dev - condition: succeeded() - steps: - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - NuGet' - inputs: - artifactName: 'drop-nuget' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Win-x86' - inputs: - artifactName: 'drop-win-x86-zip' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Arm64' - inputs: - artifactName: 'drop-nuget-arm64' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Linux' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - MacOS' - inputs: - artifactName: 'drop-osx' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - script: | - pushd $(Build.BinariesDirectory)\nuget-artifact - dir - powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/unzip.exe -OutFile unzip.exe" - powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe" - set PATH=%CD%;%PATH% - SETLOCAL EnableDelayedExpansion - FOR /R %%i IN (*.nupkg) do ( - set filename=%%~ni - IF NOT "!filename:~25,7!"=="Managed" ( - rename %%~ni.nupkg %%~ni.zip - unzip %%~ni.zip -d %%~ni - del /Q %%~ni.zip - unzip win-x86.zip -d win-x86 - unzip win10-arm.zip -d win10-arm - unzip linux-x64.zip -d linux-x64 - mkdir %%~ni\runtimes\win-x86 - mkdir %%~ni\runtimes\win10-arm - mkdir %%~ni\runtimes\linux-x64 - mkdir %%~ni\runtimes\win-x86\native - mkdir %%~ni\runtimes\win10-arm\native - mkdir %%~ni\runtimes\linux-x64\native - move win-x86\runtimes\win-x86\native\onnxruntime.dll %%~ni\runtimes\win-x86\native\onnxruntime.dll - move win-x86\runtimes\win-x86\native\onnxruntime.lib %%~ni\runtimes\win-x86\native\onnxruntime.lib - move win-x86\runtimes\win-x86\native\onnxruntime.pdb %%~ni\runtimes\win-x86\native\onnxruntime.pdb - move win10-arm\runtimes\win10-arm\native\onnxruntime.lib %%~ni\runtimes\win10-arm\native\onnxruntime.lib - move win10-arm\runtimes\win10-arm\native\onnxruntime.dll %%~ni\runtimes\win10-arm\native\onnxruntime.dll - move win10-arm\runtimes\win10-arm\native\onnxruntime.pdb %%~ni\runtimes\win10-arm\native\onnxruntime.pdb - move linux-x64\linux-x64\libonnxruntime.so %%~ni\runtimes\linux-x64\native\libonnxruntime.so - unzip osx-x64.zip -d osx-x64 - dir osx-x64 /s - mkdir %%~ni\runtimes\osx-x64 - mkdir %%~ni\runtimes\osx-x64\native - move osx-x64\osx-x64\libonnxruntime.dylib %%~ni\runtimes\osx-x64\native\libonnxruntime.dylib - move osx-x64\osx-x64\libonnxruntime.dylib.dSYM %%~ni\runtimes\osx-x64\native\libonnxruntime.dylib.dSYM - pushd %%~ni - zip -r ..\%%~ni.zip . - popd - move %%~ni.zip %%~ni.nupkg - ) - ) - popd - copy $(Build.BinariesDirectory)\nuget-artifact\*.nupkg $(Build.ArtifactStagingDirectory) - displayName: 'Bundle Native NuGet and other binaries' - - - template: ../../templates/esrp_nuget.yml - parameters: - DisplayName: 'ESRP - sign NuGet package' - FolderPath: '$(Build.ArtifactStagingDirectory)' - DoEsrp: ${{ parameters.DoEsrp }} - - - template: ../../templates/validate-nuget.yml - parameters: - NugetPath: '$(Build.ArtifactStagingDirectory)' - NugetPackage: 'Microsoft.ML.OnnxRuntime.*nupkg' - PlatformsSupported: 'win-x64,win-x86,win10-arm,linux-x64,osx-x64' - VerifyNugetSigning: ${{ parameters.DoEsrp }} - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline NuGet Artifact' - inputs: - artifactName: 'drop-signed-nuget' - targetPath: '$(Build.ArtifactStagingDirectory)' - -- template: test_all_os.yml diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-noopenmp.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-noopenmp.yml deleted file mode 100644 index 3cc942fdcd6bb..0000000000000 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-noopenmp.yml +++ /dev/null @@ -1,324 +0,0 @@ -parameters: - DoEsrp: 'false' - PackageName: 'Microsoft.ML.OnnxRuntime.NoOpenMP' - DoCompliance: 'false' - -jobs: -- template: ../../templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - ArtifactName: 'drop-nuget' - JobName: 'Windows_CI_Dev' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"' - BuildArch: 'x64' - EnvSetupScript: 'setup_env.bat' - sln_platform: 'x64' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - OrtPackageId: 'Microsoft.ML.OnnxRuntime.NoOpenMP' - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.NoOpenMP - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- template: ../../templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - ArtifactName: 'drop-win-x86-zip' - JobName: 'Windows_CI_Dev_x86' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --x86 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"' - BuildArch: 'x86' - EnvSetupScript: 'setup_env_x86.bat' - sln_platform: 'Win32' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - OrtPackageId: 'Microsoft.ML.OnnxRuntime.NoOpenMP' - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.NoOpenMP - cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\ - ren *.nupkg win-x86.zip - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- template: ../../templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - ArtifactName: 'drop-win-arm64-zip' - JobName: 'Windows_CI_Dev_arm64' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --arm64 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"' - BuildArch: 'x64' - EnvSetupScript: 'setup_env.bat' - sln_platform: 'arm64' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - OrtPackageId: 'Microsoft.ML.OnnxRuntime.NoOpenMP' - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=arm64 /p:ProtocDirectory=$(Build.BinariesDirectory)\host_protoc\Release /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.NoOpenMP - cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\ - ren *.nupkg win-arm64.zip - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- template: ../../templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - ArtifactName: 'drop-win-arm-zip' - JobName: 'Windows_CI_Dev_arm' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --arm --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"' - BuildArch: 'x64' - EnvSetupScript: 'setup_env.bat' - sln_platform: 'arm' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - OrtPackageId: 'Microsoft.ML.OnnxRuntime.NoOpenMP' - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=arm /p:ProtocDirectory=$(Build.BinariesDirectory)\host_protoc\Release /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.NoOpenMP - cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\ - ren *.nupkg win-arm.zip - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- job: 'Linux_CI_Dev' - workspace: - clean: all - timeoutInMinutes: 120 - pool: $(AgentPoolLinux) - steps: - - template: ../../templates/set-version-number-variables-step.yml - - template: ../../templates/linux-set-variables-and-download.yml - - template: ../../templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecentoscpubuild - - task: CmdLine@2 - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecentoscpubuild /bin/bash -c "python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" - workingDirectory: $(Build.SourcesDirectory) - - script: | - set -e -x - mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 - cd $(Build.BinariesDirectory)/linux-x64 - zip -r linux-x64.zip linux-x64 - cp $(Build.BinariesDirectory)/linux-x64/linux*.zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata - ls -al $(Build.ArtifactStagingDirectory) - displayName: 'Create Artifacts' - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline Artifact' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.ArtifactStagingDirectory)' - - template: ../../templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - template: ../../templates/clean-agent-build-directory-step.yml - -- template: ../../templates/mac-ci.yml - parameters: - AgentPool : $(AgentPoolMacOS) - JobName: 'MacOS_CI_Dev' - BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config RelWithDebInfo' - DoNugetPack : 'true' - NuPackScript: | - set -e -x - mkdir $(Build.BinariesDirectory)/osx-x64 - find $(Build.BinariesDirectory) - cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ - dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM - strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib - find $(Build.BinariesDirectory)/osx-x64 -ls - cwd=`pwd` - cd $(Build.BinariesDirectory) - zip -r osx-x64.zip osx-x64 - cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/RelWithDebInfo/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata - cd $cwd - -- job: NuGet_Packaging - workspace: - clean: all - pool: 'Win-CPU-2019' - dependsOn: - - Windows_CI_Dev - - Windows_CI_Dev_x86 - - Linux_CI_Dev - - MacOS_CI_Dev - condition: succeeded() - steps: - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - NuGet' - inputs: - artifactName: 'drop-nuget' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Win-x86' - inputs: - artifactName: 'drop-win-x86-zip' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Win-arm64' - inputs: - artifactName: 'drop-win-arm64-zip' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Win-arm' - inputs: - artifactName: 'drop-win-arm-zip' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Linux' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - MacOS' - inputs: - artifactName: 'drop-osx' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - template: bundle_dlls.yml - - - template: ../../templates/esrp_nuget.yml - parameters: - DisplayName: 'ESRP - sign NuGet package' - FolderPath: '$(Build.ArtifactStagingDirectory)' - DoEsrp: ${{ parameters.DoEsrp }} - - - template: ../../templates/validate-nuget.yml - parameters: - NugetPath: '$(Build.ArtifactStagingDirectory)' - NugetPackage: 'Microsoft.ML.OnnxRuntime.NoOpenMP*nupkg' - PlatformsSupported: 'win-x64,win-x86,linux-x64,osx-x64' - VerifyNugetSigning: ${{ parameters.DoEsrp }} - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline NuGet Artifact' - inputs: - artifactName: 'drop-signed-nuget' - targetPath: '$(Build.ArtifactStagingDirectory)' - - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() - -- template: test_all_os.yml - -- job: Publish_NuGet_Package_And_Report - workspace: - clean: all - variables: - - group: Dashboard_MySQL_Secret - - name: GDN_CODESIGN_TARGETDIRECTORY - value: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - pool: 'Win-CPU-2019' - condition: and (succeeded(), and (${{ parameters.DoEsrp }}, eq(variables['Build.SourceBranch'], 'refs/heads/master'))) - dependsOn: - - NuGet_Test_Win - - NuGet_Test_Linux - - NuGet_Test_MacOS - steps: - - - template: ../../templates/set-version-number-variables-step.yml - - - template: upload-binary-sizes-from-nuget-package.yml - parameters: - downloadPath: $(Build.BinariesDirectory)/nuget-artifact/final-package - gitCommitHash: $(OnnxRuntimeGitCommitHashShort) - - - template: get-nuget-package-version-as-variable.yml - parameters: - packageFolder: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - - - task: PowerShell@2 - displayName: 'Get Current Date' - inputs: - targetType: 'inline' - script: | - $date = $(Get-Date -Format "yyyy-MM-dd") - Write-Host "##vso[task.setvariable variable=CurrentDate]$date" - - - task: AzureFileCopy@3 - displayName: 'Copy Signed Native NuGet Package to Blob Store' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - sourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.NoOpenMP.$(NuGetPackageVersionNumber).nupkg' - azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' - destination: azureBlob - storage: ortpackages - containerName: ortpackages - blobPrefix: '$(CurrentDate)/' - continueOnError: true - - - task: AzureFileCopy@3 - displayName: 'Copy Signed Managed NuGet Package to Blob Store' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - sourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg' - azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' - destination: azureBlob - storage: ortpackages - containerName: ortpackages - blobPrefix: '$(CurrentDate)/' - continueOnError: true - - - task: DotNetCoreCLI@2 - displayName: 'Copy Signed Native NuGet Package to Internal NuGet Feed' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - command: 'push' - searchPatternPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.NoOpenMP.$(NuGetPackageVersionNumber).nupkg' - feedPublish: 'OnnxRuntime' - - - task: DotNetCoreCLI@2 - displayName: 'Copy Signed Managed NuGet Package to Internal NuGet Feed' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - command: 'push' - searchPatternPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg' - feedPublish: 'OnnxRuntime' - - - task: NuGetCommand@2 - displayName: 'Copy Signed Native NuGet Package to ORT-NIGHTLY' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - command: 'push' - packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.NoOpenMP.$(NuGetPackageVersionNumber).nupkg' - publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7' - - - task: NuGetCommand@2 - displayName: 'Copy Signed Managed NuGet Package to ORT-NIGHTLY' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - command: 'push' - packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg' - publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7' - - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml deleted file mode 100644 index 4a5ee41f8ddcc..0000000000000 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ /dev/null @@ -1,320 +0,0 @@ -parameters: - DoEsrp: 'false' - PackageName: 'Microsoft.ML.OnnxRuntime' - DoCompliance: 'false' - -jobs: -- template: ../../templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - ArtifactName: 'drop-nuget' - JobName: 'Windows_CI_Dev' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"' - BuildArch: 'x64' - EnvSetupScript: 'setup_env.bat' - sln_platform: 'x64' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- template: ../../templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - ArtifactName: 'drop-win-x86-zip' - JobName: 'Windows_CI_Dev_x86' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --x86 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"' - BuildArch: 'x86' - EnvSetupScript: 'setup_env_x86.bat' - sln_platform: 'Win32' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage - cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\ - ren *.nupkg win-x86.zip - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- template: ../../templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - ArtifactName: 'drop-win-arm64-zip' - JobName: 'Windows_CI_Dev_arm64' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --arm64 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"' - BuildArch: 'x64' - EnvSetupScript: 'setup_env.bat' - sln_platform: 'arm64' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=arm64 /p:ProtocDirectory=$(Build.BinariesDirectory)\host_protoc\Release /t:CreatePackage - cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\ - ren *.nupkg win-arm64.zip - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- template: ../../templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - ArtifactName: 'drop-win-arm-zip' - JobName: 'Windows_CI_Dev_arm' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --arm --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"' - BuildArch: 'x64' - EnvSetupScript: 'setup_env.bat' - sln_platform: 'arm' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=arm /p:ProtocDirectory=$(Build.BinariesDirectory)\host_protoc\Release /t:CreatePackage - cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\ - ren *.nupkg win-arm.zip - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- job: 'Linux_CI_Dev' - workspace: - clean: all - timeoutInMinutes: 120 - pool: $(AgentPoolLinux) - steps: - - template: ../../templates/set-version-number-variables-step.yml - - template: ../../templates/linux-set-variables-and-download.yml - - template: ../../templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecentoscpubuild - - task: CmdLine@2 - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecentoscpubuild /bin/bash -c "python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" - workingDirectory: $(Build.SourcesDirectory) - - script: | - set -e -x - mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 - cd $(Build.BinariesDirectory)/linux-x64 - zip -r linux-x64.zip linux-x64 - cp $(Build.BinariesDirectory)/linux-x64/linux*.zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata - ls -al $(Build.ArtifactStagingDirectory) - displayName: 'Create Artifacts' - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline Artifact' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.ArtifactStagingDirectory)' - - template: ../../templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - template: ../../templates/clean-agent-build-directory-step.yml - -- template: ../../templates/mac-ci.yml - parameters: - AgentPool : $(AgentPoolMacOS) - JobName: 'MacOS_CI_Dev' - BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --config RelWithDebInfo' - DoNugetPack : 'true' - NuPackScript: | - set -e -x - mkdir $(Build.BinariesDirectory)/osx-x64 - find $(Build.BinariesDirectory) - cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ - dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM - strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib - find $(Build.BinariesDirectory)/osx-x64 -ls - cwd=`pwd` - cd $(Build.BinariesDirectory) - zip -r osx-x64.zip osx-x64 - cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/RelWithDebInfo/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata - cd $cwd - -- job: NuGet_Packaging - workspace: - clean: all - pool: 'Win-CPU-2019' - dependsOn: - - Windows_CI_Dev - - Windows_CI_Dev_x86 - - Linux_CI_Dev - - MacOS_CI_Dev - condition: succeeded() - steps: - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - NuGet' - inputs: - artifactName: 'drop-nuget' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Win-x86' - inputs: - artifactName: 'drop-win-x86-zip' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Win-arm64' - inputs: - artifactName: 'drop-win-arm64-zip' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Win-arm' - inputs: - artifactName: 'drop-win-arm-zip' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Linux' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - MacOS' - inputs: - artifactName: 'drop-osx' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - template: bundle_dlls.yml - - - template: ../../templates/esrp_nuget.yml - parameters: - DisplayName: 'ESRP - sign NuGet package' - FolderPath: '$(Build.ArtifactStagingDirectory)' - DoEsrp: ${{ parameters.DoEsrp }} - - - template: ../../templates/validate-nuget.yml - parameters: - NugetPath: '$(Build.ArtifactStagingDirectory)' - NugetPackage: 'Microsoft.ML.OnnxRuntime.*nupkg' - PlatformsSupported: 'win-x64,win-x86,linux-x64,osx-x64' - VerifyNugetSigning: ${{ parameters.DoEsrp }} - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline NuGet Artifact' - inputs: - artifactName: 'drop-signed-nuget' - targetPath: '$(Build.ArtifactStagingDirectory)' - - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() - -- template: test_all_os.yml - -- job: Publish_NuGet_Package_And_Report - workspace: - clean: all - variables: - - group: Dashboard_MySQL_Secret - - name: GDN_CODESIGN_TARGETDIRECTORY - value: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - pool: 'Win-CPU-2019' - condition: and (succeeded(), and (${{ parameters.DoEsrp }}, eq(variables['Build.SourceBranch'], 'refs/heads/master'))) - dependsOn: - - NuGet_Test_Win - - NuGet_Test_Linux - - NuGet_Test_MacOS - steps: - - - template: ../../templates/set-version-number-variables-step.yml - - - template: upload-binary-sizes-from-nuget-package.yml - parameters: - downloadPath: $(Build.BinariesDirectory)/nuget-artifact/final-package - gitCommitHash: $(OnnxRuntimeGitCommitHashShort) - - - template: get-nuget-package-version-as-variable.yml - parameters: - packageFolder: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - - - task: PowerShell@2 - displayName: 'Get Current Date' - inputs: - targetType: 'inline' - script: | - $date = $(Get-Date -Format "yyyy-MM-dd") - Write-Host "##vso[task.setvariable variable=CurrentDate]$date" - - - task: AzureFileCopy@3 - displayName: 'Copy Signed Native NuGet Package to Blob Store' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - sourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(NuGetPackageVersionNumber).nupkg' - azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' - destination: azureBlob - storage: ortpackages - containerName: ortpackages - blobPrefix: '$(CurrentDate)/' - continueOnError: true - - - task: AzureFileCopy@3 - displayName: 'Copy Signed Managed NuGet Package to Blob Store' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - sourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg' - azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' - destination: azureBlob - storage: ortpackages - containerName: ortpackages - blobPrefix: '$(CurrentDate)/' - continueOnError: true - - - task: DotNetCoreCLI@2 - displayName: 'Copy Signed Native NuGet Package to Internal NuGet Feed' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - command: 'push' - searchPatternPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(NuGetPackageVersionNumber).nupkg' - feedPublish: 'OnnxRuntime' - - - task: DotNetCoreCLI@2 - displayName: 'Copy Signed Managed NuGet Package to Internal NuGet Feed' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - command: 'push' - searchPatternPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg' - feedPublish: 'OnnxRuntime' - - - task: NuGetCommand@2 - displayName: 'Copy Signed Native NuGet Package to ORT-NIGHTLY' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - command: 'push' - packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(NuGetPackageVersionNumber).nupkg' - publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7' - - - task: NuGetCommand@2 - displayName: 'Copy Signed Managed NuGet Package to ORT-NIGHTLY' - condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme - inputs: - command: 'push' - packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg' - publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7' - - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml index 396157ec2b207..da6e1ad16c3b0 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml @@ -1,6 +1,15 @@ -parameters: - gitCommitHash: '' - downloadPath: $(Build.BinariesDirectory)/nuget-artifact/final-package +parameters: +- name: gitCommitHash + type: string + default: '' + +- name: downloadPath + type: string + default: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + +- name: buildVariant + type: string + default: 'openmp' steps: - task: DownloadPipelineArtifact@0 @@ -36,10 +45,10 @@ steps: echo listing lib files in the zip REM use a single .csv file to put the data echo os,arch,build_config,size > binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo linux,x64,openmp,%%a >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo osx,x64,openmp,%%a >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x64,openmp,%%a >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x86,openmp,%%a >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo linux,x64,${{ parameters.buildVariant }},%%a >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo osx,x64,${{ parameters.buildVariant }},%%a >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x64,${{ parameters.buildVariant }},%%a >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x86,${{ parameters.buildVariant }},%%a >> binary_size_data.txt echo calling python script to post to database python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{ parameters.gitCommitHash }} --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) ) diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-posix.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-posix.yml index 225e630e15d2e..3ca950d20a25e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-posix.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-posix.yml @@ -1,10 +1,26 @@ # sets up common build tools for the windows build machines before build parameters: - buildConfig: 'RelWithDebInfo' - artifactName: 'onnxruntime-linux-x64' - libraryName: 'libonnxruntime.so' - commitId: '' +- name: buildConfig + type: string + default: 'RelWithDebInfo' + +- name: artifactName + type: string + default: 'onnxruntime-linux-x64' + +- name: artifactNameNoVersionString + type: string + default: 'onnxruntime-linux-x64' + +- name: libraryName + type: string + default: 'libonnxruntime.so' + +- name: commitId + type: string + default: '' + steps: - task: ShellScript@2 displayName: 'Copy build artifacts for zipping' @@ -22,7 +38,7 @@ steps: archiveFile: '$(Build.ArtifactStagingDirectory)/${{parameters.artifactName}}.tgz' replaceExistingArchive: true - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@1 inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)/${{parameters.artifactName}}.tgz' - artifactName: 'drop-${{parameters.artifactName}}' + targetPath: '$(Build.ArtifactStagingDirectory)/${{parameters.artifactName}}.tgz' + artifactName: '${{parameters.artifactNameNoVersionString}}' diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml index be1b06de823ea..a656500842d24 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml @@ -1,9 +1,30 @@ # sets up common build tools for the windows build machines before build parameters: - buildConfig: 'RelWithDebInfo' - artifactName: 'onnxruntime-win-x64' - commitId: '' +- name: DoEsrp + displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release. + type: boolean + default: true + +- name: buildConfig + displayName: buildConfig + type: string + default: 'RelWithDebInfo' + +- name: artifactName + displayName: artifactName，like 'onnxruntime-win-x64-1.6.0' + type: string + default: '' + +- name: artifactNameNoVersionString + type: string + default: 'onnxruntime-win-x64' + +- name: commitId + displayName: commitId + type: string + default: '' + steps: - task: CmdLine@2 displayName: 'Copy build artifacts for zipping' @@ -37,7 +58,7 @@ steps: parameters: FolderPath: '$(Build.BinariesDirectory)\${{parameters.artifactName}}' DisplayName: 'ESRP - Sign Native dlls' - DoEsrp: true + DoEsrp: ${{parameters.DoEsrp}} Pattern: '*.dll,*.exe' - task: ArchiveFiles@2 @@ -48,7 +69,7 @@ steps: archiveFile: '$(Build.ArtifactStagingDirectory)\${{parameters.artifactName}}.zip' replaceExistingArchive: true - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@1 inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)\${{parameters.artifactName}}.zip' - artifactName: 'drop-${{parameters.artifactName}}' + targetPath: '$(Build.ArtifactStagingDirectory)\${{parameters.artifactName}}.zip' + artifactName: '${{parameters.artifactNameNoVersionString}}' \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml new file mode 100644 index 0000000000000..e21ca73fb8234 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml @@ -0,0 +1,499 @@ +parameters: +- name: RunOnnxRuntimeTests + displayName: Run Tests? + type: boolean + default: true + +- name: DoCompliance + displayName: Run Compliance Tasks? + type: boolean + default: true + +- name: PublishingNuget + displayName: Publishing Nuget Packages and report binary size to mysql + type: boolean + default: true + +- name: DoEsrp + displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release. + type: boolean + default: false + +- name: IsReleaseBuild + displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release. + type: boolean + default: false + +- name: AdditionalBuildFlags + displayName: Additional build flags for build.py + type: string + default: '' + +- name: AdditionalWinBuildFlags + displayName: Additional build flags that just for Windows Builds + type: string + default: '' + +- name: OrtNugetPackageId + displayName: Package name for nuget + type: string + default: 'Microsoft.ML.OnnxRuntime' + +- name: BuildVariant + type: string + default: 'openmp' + +jobs: +- job: Linux_C_API_Packaging_CPU_x64 + workspace: + clean: all + timeoutInMinutes: 60 + pool: 'Linux-CPU' + steps: + - template: set-version-number-variables-step.yml + - template: get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos + Context: tools/ci_build/github/linux/docker + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" + Repository: onnxruntimecentoscpubuild + - task: CmdLine@2 + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecentoscpubuild /bin/bash -c "python3 \ + /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ + --skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-x64" + workingDirectory: $(Build.SourcesDirectory) + + - script: | + set -e -x + mkdir $(Build.ArtifactStagingDirectory)/testdata + cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata + ls -al $(Build.ArtifactStagingDirectory) + displayName: 'Create Artifacts' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline Artifact' + inputs: + artifactName: 'drop-linux' + targetPath: '$(Build.ArtifactStagingDirectory)' + + - template: c-api-artifacts-package-and-publish-steps-posix.yml + parameters: + buildConfig: 'Release' + artifactName: 'onnxruntime-linux-x64-$(OnnxRuntimeVersion)' + artifactNameNoVersionString: 'onnxruntime-linux-x64' + libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' + commitId: $(OnnxRuntimeGitCommitHash) + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + +- job: MacOS_C_API_Packaging_CPU_x64 + workspace: + clean: all + pool: + vmImage: 'macOS-10.14' + timeoutInMinutes: 120 + steps: + - template: set-version-number-variables-step.yml + + - script: | + set -e + pushd . + cd $(Build.SourcesDirectory)/cmake/external/protobuf + cmake ./cmake -DCMAKE_INSTALL_PREFIX=$(Build.BinariesDirectory)/protobuf -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo + make -j$(getconf _NPROCESSORS_ONLN) + make install + popd + export PATH=$(Build.BinariesDirectory)/protobuf/bin:$PATH + export ONNX_ML=1 + export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" + sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' + sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer + brew install libomp + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release + displayName: 'Build and Test MacOS' + - template: c-api-artifacts-package-and-publish-steps-posix.yml + parameters: + buildConfig: 'Release' + artifactName: 'onnxruntime-osx-x64-$(OnnxRuntimeVersion)' + artifactNameNoVersionString: 'onnxruntime-osx-x64' + libraryName: 'libonnxruntime.$(OnnxRuntimeVersion).dylib' + commitId: $(OnnxRuntimeGitCommitHash) + + - script: | + set -e -x + mkdir $(Build.ArtifactStagingDirectory)/testdata + cp $(Build.BinariesDirectory)/Release/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata + displayName: 'Create Artifacts' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline Artifact' + inputs: + artifactName: 'drop-osx' + targetPath: '$(Build.ArtifactStagingDirectory)' + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + + +- template: win-ci.yml + parameters: + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + OrtNugetPackageId: ${{ parameters.OrtNugetPackageId }} + job_name_suffix: x86_openmp + EnvSetupScript: setup_env_x86.bat + buildArch: x86 + msbuildPlatform: Win32 + packageName: x86 + buildparameter: --x86 ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}} + runTests: ${{ parameters.RunOnnxRuntimeTests }} + buildJava: false + +- template: win-ci.yml + parameters: + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + OrtNugetPackageId: ${{ parameters.OrtNugetPackageId }} + job_name_suffix: arm_openmp + EnvSetupScript: setup_env.bat + buildArch: x64 + msbuildPlatform: arm + packageName: arm + buildparameter: --arm ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}} + runTests: false + buildJava: false + +- template: win-ci.yml + parameters: + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + OrtNugetPackageId: ${{ parameters.OrtNugetPackageId }} + job_name_suffix: arm64_openmp + EnvSetupScript: setup_env.bat + buildArch: x64 + msbuildPlatform: arm64 + packageName: arm64 + buildparameter: --arm64 ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}} + runTests: false + buildJava: false + +- template: win-ci.yml + parameters: + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + OrtNugetPackageId: ${{ parameters.OrtNugetPackageId }} + job_name_suffix: x64_openmp + EnvSetupScript: setup_env.bat + buildArch: x64 + msbuildPlatform: x64 + packageName: x64 + buildparameter: ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}} + runTests: ${{ parameters.RunOnnxRuntimeTests }} + buildJava: false + + +- job: NuGet_Packaging + workspace: + clean: all + pool: 'Win-CPU-2019' + variables: + OrtPackageId: ${{ parameters.OrtNugetPackageId }} + breakCodesignValidationInjection: ${{ parameters.DoEsrp }} + dependsOn: + - Linux_C_API_Packaging_CPU_x64 + - MacOS_C_API_Packaging_CPU_x64 + - Windows_Packaging_CPU_x86_openmp + - Windows_Packaging_CPU_x64_openmp + - Windows_Packaging_CPU_arm_openmp + - Windows_Packaging_CPU_arm64_openmp + condition: succeeded() + steps: + - checkout: self + submodules: true + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'onnxruntime-win-x64' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'onnxruntime-win-x86' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'onnxruntime-win-arm64' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'onnxruntime-win-arm' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'onnxruntime-osx-x64' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'onnxruntime-linux-x64' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'drop-extra' + targetPath: '$(Build.BinariesDirectory)/extra-artifact' + + #Reconstruct the build dir + - task: PowerShell@2 + displayName: 'PowerShell Script' + inputs: + targetType: filePath + filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files.ps1 + + - script: | + dir + workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact' + displayName: 'List artifacts' + + - script: | + mklink /D /J models C:\local\models + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Create models link' + + - task: NuGetToolInstaller@0 + displayName: Use Nuget 5.7.0 + inputs: + versionSpec: 5.7.0 + + - task: DotNetCoreCLI@2 + displayName: 'Restore nuget packages' + inputs: + command: restore + projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + configuration: RelWithDebInfo + arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: DotNetCoreCLI@2 + displayName: 'Build C#' + inputs: + command: build + projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + configuration: RelWithDebInfo + arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - template: win-esrp-dll.yml + parameters: + FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo' + DisplayName: 'ESRP - Sign C# dlls' + DoEsrp: ${{ parameters.DoEsrp }} + + - task: DotNetCoreCLI@2 + displayName: 'Build Nuget Packages' + inputs: + command: build + projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj' + configuration: RelWithDebInfo + arguments: '--configuration RelWithDebInfo -t:CreatePackage -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: BatchScript@1 + displayName: 'Bundle Native NuGet and other binaries' + inputs: + filename: $(Build.SourcesDirectory)\tools\ci_build\github\windows\bundle_dlls.bat + workingFolder: $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts + + - task: CopyFiles@2 + displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts' + Contents: '*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - task: CopyFiles@2 + displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo' + Contents: '*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - template: esrp_nuget.yml + parameters: + DisplayName: 'ESRP - sign NuGet package' + FolderPath: '$(Build.ArtifactStagingDirectory)' + DoEsrp: ${{ parameters.DoEsrp }} + + - template: validate-nuget.yml + parameters: + NugetPath: '$(Build.ArtifactStagingDirectory)' + NugetPackage: 'Microsoft.ML.OnnxRuntime.*nupkg' + PlatformsSupported: 'win-x64,win-x86,linux-x64,osx-x64' + VerifyNugetSigning: false + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline NuGet Artifact' + inputs: + artifactName: 'drop-signed-nuget' + targetPath: '$(Build.ArtifactStagingDirectory)' + + - powershell: | + $counter = 0 + Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*.nupkg | foreach {7z x -y -o$counter $_.fullname;$counter++} + workingDirectory: '$(Build.ArtifactStagingDirectory)' + displayName: 'Unzip files to sanitize the PE files' + + - task: DotNetCoreCLI@2 + displayName: 'Build C#' + inputs: + command: custom + custom: clean + projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + configuration: RelWithDebInfo + arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: BinSkim@3 + displayName: 'Run BinSkim' + + - task: RoslynAnalyzers@2 + displayName: 'Run Roslyn Analyzers' + inputs: + userProvideBuildInfo: msBuildInfo + msBuildArchitecture: DotNetCore + msBuildCommandline: '"C:\Program Files\dotnet\dotnet.exe" build $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln --configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)' + condition: and(succeeded(), eq('${{ parameters.DoCompliance }}', true)) + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + +- template: ../nuget/templates/test_all_os.yml + +- job: Publish_NuGet_Package_And_Report + workspace: + clean: all + variables: + - group: Dashboard_MySQL_Secret + - name: GDN_CODESIGN_TARGETDIRECTORY + value: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + pool: 'Win-CPU-2019' + condition: and (succeeded(), and (${{ parameters.PublishingNuget }}, and (${{ parameters.DoEsrp }}, eq(variables['Build.SourceBranch'], 'refs/heads/master')))) + dependsOn: + - NuGet_Test_Win + - NuGet_Test_Linux + - NuGet_Test_MacOS + steps: + + - template: set-version-number-variables-step.yml + + - template: ../nuget/templates/upload-binary-sizes-from-nuget-package.yml + parameters: + downloadPath: $(Build.BinariesDirectory)/nuget-artifact/final-package + gitCommitHash: $(OnnxRuntimeGitCommitHashShort) + buildVariant: ${{ parameters.BuildVariant }} + + - template: ../nuget/templates/get-nuget-package-version-as-variable.yml + parameters: + packageFolder: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + + - task: PowerShell@2 + displayName: 'Get Current Date' + inputs: + targetType: 'inline' + script: | + $date = $(Get-Date -Format "yyyy-MM-dd") + Write-Host "##vso[task.setvariable variable=CurrentDate]$date" + + - task: AzureFileCopy@3 + displayName: 'Copy Signed Native NuGet Package to Blob Store' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme + inputs: + sourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/${{ parameters.OrtNugetPackageId }}.$(NuGetPackageVersionNumber).nupkg' + azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' + destination: azureBlob + storage: ortpackages + containerName: ortpackages + blobPrefix: '$(CurrentDate)/' + continueOnError: true + + - task: AzureFileCopy@3 + displayName: 'Copy Signed Managed NuGet Package to Blob Store' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme + inputs: + sourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg' + azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' + destination: azureBlob + storage: ortpackages + containerName: ortpackages + blobPrefix: '$(CurrentDate)/' + continueOnError: true + + - task: DotNetCoreCLI@2 + displayName: 'Copy Signed Native NuGet Package to Internal NuGet Feed' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme + inputs: + command: 'push' + searchPatternPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/${{ parameters.OrtNugetPackageId }}.$(NuGetPackageVersionNumber).nupkg' + feedPublish: 'OnnxRuntime' + + - task: DotNetCoreCLI@2 + displayName: 'Copy Signed Managed NuGet Package to Internal NuGet Feed' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme + inputs: + command: 'push' + searchPatternPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg' + feedPublish: 'OnnxRuntime' + + - task: NuGetCommand@2 + displayName: 'Copy Signed Native NuGet Package to ORT-NIGHTLY' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme + inputs: + command: 'push' + packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/${{ parameters.OrtNugetPackageId }}.$(NuGetPackageVersionNumber).nupkg' + publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7' + + - task: NuGetCommand@2 + displayName: 'Copy Signed Managed NuGet Package to ORT-NIGHTLY' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme + inputs: + command: 'push' + packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg' + publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7' + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml new file mode 100644 index 0000000000000..3e29fcedee484 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -0,0 +1,713 @@ +parameters: +- name: RunOnnxRuntimeTests + displayName: Run Tests? + type: boolean + default: true + +- name: DoCompliance + displayName: Run Compliance Tasks? + type: boolean + default: true + +- name: PublishingNuget + displayName: Publishing Nuget Packages and report binary size to mysql + type: boolean + default: true + +- name: DoEsrp + displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release. + type: boolean + default: false + +- name: IsReleaseBuild + displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release. + type: boolean + default: false + +- name: AdditionalBuildFlags + displayName: Additional build flags for build.py + type: string + default: '' + +- name: AdditionalWinBuildFlags + displayName: Additional build flags that just for Windows Builds + type: string + default: '' + +- name: OrtNugetPackageId + displayName: Package name for nuget + type: string + default: 'Microsoft.ML.OnnxRuntime' + +- name: BuildVariant + type: string + default: 'openmp' + +jobs: +- job: Linux_C_API_Packaging_CPU_x64 + workspace: + clean: all + timeoutInMinutes: 60 + pool: 'Linux-CPU' + steps: + - template: set-version-number-variables-step.yml + - template: get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos + Context: tools/ci_build/github/linux/docker + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" + Repository: onnxruntimecentoscpubuild + - task: CmdLine@2 + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecentoscpubuild /bin/bash -c "python3 \ + /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \ + --skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-x64" + workingDirectory: $(Build.SourcesDirectory) + + - script: | + set -e -x + mkdir $(Build.ArtifactStagingDirectory)/testdata + cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata + ls -al $(Build.ArtifactStagingDirectory) + displayName: 'Create Artifacts' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline Artifact' + inputs: + artifactName: 'drop-linux' + targetPath: '$(Build.ArtifactStagingDirectory)' + + - template: java-api-artifacts-package-and-publish-steps-posix.yml + parameters: + arch: 'linux-x64' + buildConfig: 'Release' + artifactName: 'onnxruntime-java-linux-x64' + version: '$(OnnxRuntimeVersion)' + libraryName: 'libonnxruntime.so' + nativeLibraryName: 'libonnxruntime4j_jni.so' + + - template: c-api-artifacts-package-and-publish-steps-posix.yml + parameters: + buildConfig: 'Release' + artifactName: 'onnxruntime-linux-x64-$(OnnxRuntimeVersion)' + artifactNameNoVersionString: 'onnxruntime-linux-x64' + libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' + commitId: $(OnnxRuntimeGitCommitHash) + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + +- job: MacOS_C_API_Packaging_CPU_x64 + workspace: + clean: all + pool: + vmImage: 'macOS-10.14' + timeoutInMinutes: 120 + steps: + - template: set-version-number-variables-step.yml + + - script: | + set -e + pushd . + cd $(Build.SourcesDirectory)/cmake/external/protobuf + cmake ./cmake -DCMAKE_INSTALL_PREFIX=$(Build.BinariesDirectory)/protobuf -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo + make -j$(getconf _NPROCESSORS_ONLN) + make install + popd + export PATH=$(Build.BinariesDirectory)/protobuf/bin:$PATH + export ONNX_ML=1 + export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" + sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' + sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer + brew install libomp + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_java ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release + displayName: 'Build and Test MacOS' + - template: c-api-artifacts-package-and-publish-steps-posix.yml + parameters: + buildConfig: 'Release' + artifactName: 'onnxruntime-osx-x64-$(OnnxRuntimeVersion)' + artifactNameNoVersionString: 'onnxruntime-osx-x64' + libraryName: 'libonnxruntime.$(OnnxRuntimeVersion).dylib' + commitId: $(OnnxRuntimeGitCommitHash) + + - script: | + set -e -x + mkdir $(Build.ArtifactStagingDirectory)/testdata + cp $(Build.BinariesDirectory)/Release/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata + displayName: 'Create Artifacts' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline Artifact' + inputs: + artifactName: 'drop-osx' + targetPath: '$(Build.ArtifactStagingDirectory)' + + - template: java-api-artifacts-package-and-publish-steps-posix.yml + parameters: + arch: 'osx-x64' + buildConfig: 'Release' + artifactName: 'onnxruntime-java-osx-x64' + version: '$(OnnxRuntimeVersion)' + libraryName: 'libonnxruntime.dylib' + nativeLibraryName: 'libonnxruntime4j_jni.dylib' + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + + +- template: win-ci.yml + parameters: + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + OrtNugetPackageId: ${{ parameters.OrtNugetPackageId }} + job_name_suffix: x86_openmp + EnvSetupScript: setup_env_x86.bat + buildArch: x86 + msbuildPlatform: Win32 + packageName: x86 + buildparameter: --x86 ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}} + runTests: ${{ parameters.RunOnnxRuntimeTests }} + buildJava: false + +- template: win-ci.yml + parameters: + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + OrtNugetPackageId: ${{ parameters.OrtNugetPackageId }} + job_name_suffix: arm_openmp + EnvSetupScript: setup_env.bat + buildArch: x64 + msbuildPlatform: arm + packageName: arm + buildparameter: --arm ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}} + runTests: false + buildJava: false + +- template: win-ci.yml + parameters: + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + OrtNugetPackageId: ${{ parameters.OrtNugetPackageId }} + job_name_suffix: arm64_openmp + EnvSetupScript: setup_env.bat + buildArch: x64 + msbuildPlatform: arm64 + packageName: arm64 + buildparameter: --arm64 ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}} + runTests: false + buildJava: false + +- template: win-ci.yml + parameters: + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + OrtNugetPackageId: ${{ parameters.OrtNugetPackageId }} + job_name_suffix: x64_openmp + EnvSetupScript: setup_env.bat + buildArch: x64 + msbuildPlatform: x64 + packageName: x64 + buildparameter: --build_java ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}} + runTests: ${{ parameters.RunOnnxRuntimeTests }} + buildJava: true + + +- job: Jar_Packaging + workspace: + clean: all + pool: 'Win-CPU-2019' + dependsOn: + - Linux_C_API_Packaging_CPU_x64 + - MacOS_C_API_Packaging_CPU_x64 + - Windows_Packaging_CPU_x86_openmp + - Windows_Packaging_CPU_x64_openmp + - Windows_Packaging_CPU_arm_openmp + - Windows_Packaging_CPU_arm64_openmp + condition: succeeded() + steps: + - checkout: self + submodules: false + - template: set-version-number-variables-step.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Pipeline Artifact - Win x64' + inputs: + buildType: 'current' + artifactName: 'drop-onnxruntime-java-win-x64' + targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Pipeline Artifact - Linux x64' + inputs: + buildType: 'current' + artifactName: 'drop-onnxruntime-java-linux-x64' + targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Pipeline Artifact - MacOS x64' + inputs: + buildType: 'current' + artifactName: 'drop-onnxruntime-java-osx-x64' + targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x64' + + - task: PowerShell@2 + displayName: 'PowerShell Script' + inputs: + targetType: filePath + filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\jar_packaging.ps1 + failOnStderr: true + showWarnings: true + workingDirectory: '$(Build.BinariesDirectory)\java-artifact' + + + + - task: CopyFiles@2 + displayName: 'Copy Java Files to Artifact Staging Directory' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Pipeline Artifact' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)' + artifact: 'onnxruntime-java' + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + +- job: NuGet_Packaging + workspace: + clean: all + pool: 'Win-CPU-2019' + variables: + OrtPackageId: ${{ parameters.OrtNugetPackageId }} + breakCodesignValidationInjection: ${{ parameters.DoEsrp }} + dependsOn: + - Linux_C_API_Packaging_CPU_x64 + - MacOS_C_API_Packaging_CPU_x64 + - Windows_Packaging_CPU_x86_openmp + - Windows_Packaging_CPU_x64_openmp + - Windows_Packaging_CPU_arm_openmp + - Windows_Packaging_CPU_arm64_openmp + condition: succeeded() + steps: + - checkout: self + submodules: true + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'onnxruntime-win-x64' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'onnxruntime-win-x86' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'onnxruntime-win-arm64' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'onnxruntime-win-arm' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'onnxruntime-osx-x64' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'onnxruntime-linux-x64' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'drop-extra' + targetPath: '$(Build.BinariesDirectory)/extra-artifact' + + #Reconstruct the build dir + - task: PowerShell@2 + displayName: 'PowerShell Script' + inputs: + targetType: filePath + filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files.ps1 + + - script: | + dir + workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact' + displayName: 'List artifacts' + + - script: | + mklink /D /J models C:\local\models + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Create models link' + + - task: NuGetToolInstaller@0 + displayName: Use Nuget 5.7.0 + inputs: + versionSpec: 5.7.0 + + - task: DotNetCoreCLI@2 + displayName: 'Restore nuget packages' + inputs: + command: restore + projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + configuration: RelWithDebInfo + arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: DotNetCoreCLI@2 + displayName: 'Build C#' + inputs: + command: build + projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + configuration: RelWithDebInfo + arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - template: win-esrp-dll.yml + parameters: + FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo' + DisplayName: 'ESRP - Sign C# dlls' + DoEsrp: ${{ parameters.DoEsrp }} + + - task: DotNetCoreCLI@2 + displayName: 'Build Nuget Packages' + inputs: + command: build + projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj' + configuration: RelWithDebInfo + arguments: '--configuration RelWithDebInfo -t:CreatePackage -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: BatchScript@1 + displayName: 'Bundle Native NuGet and other binaries' + inputs: + filename: $(Build.SourcesDirectory)\tools\ci_build\github\windows\bundle_dlls.bat + workingFolder: $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts + + - task: CopyFiles@2 + displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts' + Contents: '*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - task: CopyFiles@2 + displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo' + Contents: '*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - template: esrp_nuget.yml + parameters: + DisplayName: 'ESRP - sign NuGet package' + FolderPath: '$(Build.ArtifactStagingDirectory)' + DoEsrp: ${{ parameters.DoEsrp }} + + - template: validate-nuget.yml + parameters: + NugetPath: '$(Build.ArtifactStagingDirectory)' + NugetPackage: 'Microsoft.ML.OnnxRuntime.*nupkg' + PlatformsSupported: 'win-x64,win-x86,linux-x64,osx-x64' + VerifyNugetSigning: false + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline NuGet Artifact' + inputs: + artifactName: 'drop-signed-nuget' + targetPath: '$(Build.ArtifactStagingDirectory)' + + - powershell: | + $counter = 0 + Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*.nupkg | foreach {7z x -y -o$counter $_.fullname;$counter++} + workingDirectory: '$(Build.ArtifactStagingDirectory)' + displayName: 'Unzip files to sanitize the PE files' + + - task: DotNetCoreCLI@2 + displayName: 'Build C#' + inputs: + command: custom + custom: clean + projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + configuration: RelWithDebInfo + arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: BinSkim@3 + displayName: 'Run BinSkim' + + - task: RoslynAnalyzers@2 + displayName: 'Run Roslyn Analyzers' + inputs: + userProvideBuildInfo: msBuildInfo + msBuildArchitecture: DotNetCore + msBuildCommandline: '"C:\Program Files\dotnet\dotnet.exe" build $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln --configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)' + condition: and(succeeded(), eq('${{ parameters.DoCompliance }}', true)) + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + +- template: ../nuget/templates/test_all_os.yml + +- job: Publish_NuGet_Package_And_Report + workspace: + clean: all + variables: + - group: Dashboard_MySQL_Secret + - name: GDN_CODESIGN_TARGETDIRECTORY + value: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + pool: 'Win-CPU-2019' + condition: and (succeeded(), and (${{ parameters.PublishingNuget }}, and (${{ parameters.DoEsrp }}, eq(variables['Build.SourceBranch'], 'refs/heads/master')))) + dependsOn: + - NuGet_Test_Win + - NuGet_Test_Linux + - NuGet_Test_MacOS + steps: + + - template: set-version-number-variables-step.yml + + - template: ../nuget/templates/upload-binary-sizes-from-nuget-package.yml + parameters: + downloadPath: $(Build.BinariesDirectory)/nuget-artifact/final-package + gitCommitHash: $(OnnxRuntimeGitCommitHashShort) + buildVariant: ${{ parameters.BuildVariant }} + + - template: ../nuget/templates/get-nuget-package-version-as-variable.yml + parameters: + packageFolder: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + + - task: PowerShell@2 + displayName: 'Get Current Date' + inputs: + targetType: 'inline' + script: | + $date = $(Get-Date -Format "yyyy-MM-dd") + Write-Host "##vso[task.setvariable variable=CurrentDate]$date" + + - task: AzureFileCopy@3 + displayName: 'Copy Signed Native NuGet Package to Blob Store' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme + inputs: + sourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/${{ parameters.OrtNugetPackageId }}.$(NuGetPackageVersionNumber).nupkg' + azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' + destination: azureBlob + storage: ortpackages + containerName: ortpackages + blobPrefix: '$(CurrentDate)/' + continueOnError: true + + - task: AzureFileCopy@3 + displayName: 'Copy Signed Managed NuGet Package to Blob Store' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme + inputs: + sourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg' + azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' + destination: azureBlob + storage: ortpackages + containerName: ortpackages + blobPrefix: '$(CurrentDate)/' + continueOnError: true + + - task: DotNetCoreCLI@2 + displayName: 'Copy Signed Native NuGet Package to Internal NuGet Feed' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme + inputs: + command: 'push' + searchPatternPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/${{ parameters.OrtNugetPackageId }}.$(NuGetPackageVersionNumber).nupkg' + feedPublish: 'OnnxRuntime' + + - task: DotNetCoreCLI@2 + displayName: 'Copy Signed Managed NuGet Package to Internal NuGet Feed' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme + inputs: + command: 'push' + searchPatternPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg' + feedPublish: 'OnnxRuntime' + + - task: NuGetCommand@2 + displayName: 'Copy Signed Native NuGet Package to ORT-NIGHTLY' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme + inputs: + command: 'push' + packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/${{ parameters.OrtNugetPackageId }}.$(NuGetPackageVersionNumber).nupkg' + publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7' + + - task: NuGetCommand@2 + displayName: 'Copy Signed Managed NuGet Package to ORT-NIGHTLY' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme + inputs: + command: 'push' + packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg' + publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7' + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + +- job: Final_Jar_Testing_Windows + workspace: + clean: all + pool: 'Win-CPU-2019' + timeoutInMinutes: 60 + variables: + - name: runCodesignValidationInjection + value: false + dependsOn: + Jar_Packaging + steps: + - template: set-version-number-variables-step.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Final Jar' + inputs: + buildType: 'current' + artifactName: 'onnxruntime-java' + targetPath: '$(Build.BinariesDirectory)\final-jar' + + - task: CmdLine@2 + inputs: + script: | + mkdir test + pushd test + jar xf $(Build.BinariesDirectory)\final-jar\testing.jar + popd + powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar" + powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -OutFile protobuf-java-3.9.2.jar" + java -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.9.2.jar;onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner + workingDirectory: '$(Build.BinariesDirectory)\final-jar' + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + +- job: Final_Jar_Testing_Linux + workspace: + clean: all + pool: 'Linux-CPU' + variables: + - name: runCodesignValidationInjection + value: false + timeoutInMinutes: 60 + dependsOn: + Jar_Packaging + steps: + - template: set-version-number-variables-step.yml + - task: DownloadPipelineArtifact@2 + displayName: 'Download Final Jar' + inputs: + buildType: 'current' + artifactName: 'onnxruntime-java' + targetPath: '$(Build.BinariesDirectory)/final-jar' + + - task: CmdLine@2 + inputs: + script: | + sudo apt-get install -y openjdk-8-jdk + sudo apt autoremove + PATH=/usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin:${PATH} + echo "Java Version" + java --version + mkdir test + pushd test + jar xf $(Build.BinariesDirectory)/final-jar/testing.jar + popd + wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./ + wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./ + LD_LIBRARY_PATH=./test:${LD_LIBRARY_PATH} + java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.9.2.jar:./onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner + workingDirectory: '$(Build.BinariesDirectory)/final-jar' + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + +- job: Final_Jar_Testing_MacOs + workspace: + clean: all + pool: + vmImage: 'macOS-10.14' + variables: + - name: runCodesignValidationInjection + value: false + timeoutInMinutes: 60 + dependsOn: + Jar_Packaging + steps: + - template: set-version-number-variables-step.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Final Jar' + inputs: + buildType: 'current' + artifactName: 'onnxruntime-java' + targetPath: '$(Build.BinariesDirectory)/final-jar' + + - task: CmdLine@2 + inputs: + script: | + echo "Java Version" + java --version + mkdir test + pushd test + jar xf $(Build.BinariesDirectory)/final-jar/testing.jar + popd + wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./ + wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./ + sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer + brew install libomp + DYLD_LIBRARY_PATH=./test:${DYLD_LIBRARY_PATH} + java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.9.2.jar:./onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner + workingDirectory: '$(Build.BinariesDirectory)/final-jar' + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/compliance.yml b/tools/ci_build/github/azure-pipelines/templates/compliance.yml index 87e3c21bfa6e7..31b88e0ceaa17 100644 --- a/tools/ci_build/github/azure-pipelines/templates/compliance.yml +++ b/tools/ci_build/github/azure-pipelines/templates/compliance.yml @@ -1,14 +1,21 @@ +parameters: +- name: msbuildPlatform + displayName: msbuildPlatform + type: string + default: x64 + steps: - task: CredScan@2 displayName: 'Run CredScan' inputs: + toolMajorVersion: V2 debugMode: false continueOnError: true - task: BinSkim@3 displayName: 'Run BinSkim' inputs: - arguments: 'analyze $(Build.BinariesDirectory)\RelWithDebInfo\*.dll --recurse --verbose' + arguments: 'analyze $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.dll --recurse --verbose' continueOnError: true - task: DeleteFiles@1 @@ -20,8 +27,14 @@ steps: **/*.pdb **/*.dll +#Manually set msBuildCommandline so that we can also set CAExcludePath - task: securedevelopmentteam.vss-secure-development-tools.build-task-prefast.SDLNativeRules@2 displayName: 'Run the PREfast SDL Native Rules for MSBuild' + inputs: + userProvideBuildInfo: msBuildInfo + msBuildVersion: 16.0 + msBuildArchitecture: x64 + msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="${{parameters.msbuildPlatform}}" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64' continueOnError: true - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1 @@ -36,3 +49,9 @@ steps: displayName: 'Publish Security Analysis Logs' continueOnError: true +- task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@1 + condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + displayName: 'TSA Upload' + inputs: + tsaVersion: TsaV2 + codeBaseName: 'onnxruntime_master' \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-posix.yml b/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-posix.yml index ed020758fae3e..6b469c1a2a551 100644 --- a/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-posix.yml +++ b/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-posix.yml @@ -13,17 +13,9 @@ steps: inputs: scriptPath: 'tools/ci_build/github/linux/java_copy_strip_binary.sh' args: '-r $(Build.BinariesDirectory) -c ${{parameters.buildConfig}} -a ${{parameters.artifactName}} -l ${{parameters.libraryName}} -n ${{parameters.nativeLibraryName}} -v ${{parameters.version}} -h ${{parameters.arch}}' - workingDirectory: '$(Build.BinariesDirectory)/${{parameters.buildConfig}}' - - - task: ArchiveFiles@2 - inputs: - rootFolderOrFile: '$(Build.BinariesDirectory)/${{parameters.artifactName}}' - includeRootFolder: true - archiveType: 'zip' # Options: zip, 7z, tar, wim - archiveFile: '$(Build.ArtifactStagingDirectory)/${{parameters.artifactName}}.zip' - replaceExistingArchive: true + workingDirectory: '$(Build.BinariesDirectory)/${{parameters.buildConfig}}' - task: PublishBuildArtifacts@1 inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)/${{parameters.artifactName}}.zip' + pathtoPublish: '$(Build.BinariesDirectory)/${{parameters.artifactName}}' artifactName: 'drop-${{parameters.artifactName}}' diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml index 67d5e4dca97f2..6223c3c129145 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml @@ -266,9 +266,10 @@ jobs: displayName: 'Unzip package to test' workingDirectory: '$(Build.ArtifactStagingDirectory)' - # Compliance tasks require logs from Debug Build - ${{ if eq(parameters['DoCompliance'], 'true') }}: - template: compliance.yml + parameters : + msbuildPlatform: ${{ parameters.sln_platform }} - template: component-governance-component-detection-steps.yml parameters : diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml new file mode 100644 index 0000000000000..bf619d0fe1278 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -0,0 +1,251 @@ +parameters: +- name: DoCompliance + displayName: Run Compliance Tasks? + type: boolean + default: true + +- name: DoEsrp + displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release. + type: boolean + default: false + +- name: IsReleaseBuild + displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release. + type: boolean + default: false + +- name: OrtNugetPackageId + displayName: Package name for nuget + type: string + default: 'Microsoft.ML.OnnxRuntime' + +- name: EnvSetupScript + displayName: Package name for nuget + type: string + default: 'Microsoft.ML.OnnxRuntime' + +- name: buildArch + displayName: Package name for nuget + type: string + default: 'Microsoft.ML.OnnxRuntime' + +- name: msbuildPlatform + displayName: Package name for nuget + type: string + default: 'Microsoft.ML.OnnxRuntime' + +- name: packageName + displayName: Package name for nuget + type: string + default: 'Microsoft.ML.OnnxRuntime' + +- name: buildparameter + displayName: Package name for nuget + type: string + default: 'Microsoft.ML.OnnxRuntime' + +- name: runTests + displayName: Package name for nuget + type: string + default: 'Microsoft.ML.OnnxRuntime' + +- name: buildJava + displayName: Package name for nuget + type: string + default: 'Microsoft.ML.OnnxRuntime' + +- name: job_name_suffix + displayName: job name for nuget + type: string + default: '' + +- name: ort_build_pool_name + displayName: job name for nuget + type: string + default: 'Win-CPU-2019' + +jobs: +- job: Windows_Packaging_CPU_${{ parameters.job_name_suffix }} + workspace: + clean: all + pool: ${{ parameters.ort_build_pool_name }} + timeoutInMinutes: 300 + steps: + - template: telemetry-steps.yml + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: ${{ parameters.buildArch }} + + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + + - script: | + python -m pip install -q pyopenssl setuptools wheel numpy + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' + - powershell: | + $Env:USE_MSVC_STATIC_RUNTIME=1 + $Env:ONNX_ML=1 + $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{ parameters.buildArch }}-windows-static" + python setup.py bdist_wheel + python -m pip uninstall -y onnx + Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} + workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx' + displayName: 'Install ONNX' + + - template: set-test-data-variables-step.yml + - template: set-version-number-variables-step.yml + + - task: PythonScript@0 + displayName: 'Generate cmake config' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}' + workingDirectory: '$(Build.BinariesDirectory)' + + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' + platform: ${{ parameters.msbuildPlatform }} + configuration: RelWithDebInfo + msbuildArchitecture: ${{ parameters.buildArch }} + maximumCpuCount: true + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + createLogFile: true + + - task: PythonScript@0 + displayName: 'test' + condition: and(succeeded(), eq('${{ parameters.runTests}}', true)) + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}' + workingDirectory: '$(Build.BinariesDirectory)' + + - script: | + dir *.dll + mkdir $(Build.ArtifactStagingDirectory)\testdata + workingDirectory: '$(Build.BinariesDirectory)/RelWithDebInfo/RelWithDebInfo' + displayName: 'List built DLLs' + + - template: c-api-artifacts-package-and-publish-steps-windows.yml + parameters: + buildConfig: RelWithDebInfo + artifactName: 'onnxruntime-win-${{ parameters.packageName }}-$(OnnxRuntimeVersion)' + artifactNameNoVersionString: 'onnxruntime-win-${{ parameters.packageName }}' + commitId: $(OnnxRuntimeGitCommitHash) + DoEsrp: ${{ parameters.DoEsrp }} + + #Upload protoc.exe, which will be used in nuget build for generating C# files + - task: PublishPipelineArtifact@1 + condition: and(succeeded(), eq('${{ parameters.packageName}}', 'x64')) + inputs: + targetPath: '$(Build.BinariesDirectory)\RelWithDebInfo\external\protobuf\cmake\RelWithDebInfo\protoc.exe' + artifactName: 'drop-extra' + + + - task: CopyFiles@2 + displayName: 'Copy custom_op_library to: $(Build.ArtifactStagingDirectory)' + condition: and(succeeded(), eq('${{ parameters.packageName}}', 'x64')) + inputs: + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' + Contents: 'custom_op_library.dll' + TargetFolder: '$(Build.ArtifactStagingDirectory)/testdata' + + #To be used in test_win.yml + - task: PublishPipelineArtifact@1 + condition: and(succeeded(), eq('${{ parameters.packageName}}', 'x64')) + inputs: + targetPath: '$(Build.BinariesDirectory)\RelWithDebInfo\external\protobuf\cmake\RelWithDebInfo\protoc.exe' + artifactName: 'drop-nuget' + + + - task: CmdLine@2 + condition: and(succeeded(), eq('${{ parameters.buildJava}}', true)) + inputs: + script: | + @echo on + cd $(Build.SourcesDirectory)\java + gradle cmakeCheck -DcmakeBuildDir=$(Build.BinariesDirectory)\RelWithDebInfo + if %errorlevel% neq 0 exit /b %errorlevel% + cd $(Build.BinariesDirectory)\RelWithDebInfo + set NATIVE_FOLDER=$(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\stage\ai\onnxruntime\native\win-x64 + mkdir %NATIVE_FOLDER% + echo "Directories created" + copy .\java\build\libs\*.jar $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }} + pushd $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }} + set artifact_id=onnxruntime + jar xf onnxruntime-$(OnnxRuntimeVersion).jar META-INF\maven\com.microsoft.onnxruntime\%artifact_id%\pom.xml + move META-INF\maven\com.microsoft.onnxruntime\%artifact_id%\pom.xml onnxruntime-$(OnnxRuntimeVersion).pom + rd /s /q META-INF + popd + copy .\RelWithDebInfo\onnxruntime.pdb %NATIVE_FOLDER% + copy .\RelWithDebInfo\onnxruntime4j_jni.pdb %NATIVE_FOLDER% + copy $(Build.SourcesDirectory)\docs\Privacy.md $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\stage\Privacy.md + copy $(Build.SourcesDirectory)\ThirdPartyNotices.txt $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\stage\ThirdPartyNotices.txt + @echo $(OnnxRuntimeGitCommitHash) > $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\stage\GIT_COMMIT_ID + pushd $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\stage + jar uf $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\onnxruntime-$(OnnxRuntimeVersion).jar ai\onnxruntime\native\win-x64\onnxruntime.pdb + jar uf $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\onnxruntime-$(OnnxRuntimeVersion).jar ai\onnxruntime\native\win-x64\onnxruntime4j_jni.pdb + jar uf $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\onnxruntime-$(OnnxRuntimeVersion).jar Privacy.md ThirdPartyNotices.txt GIT_COMMIT_ID + popd + pushd $(Build.SourcesDirectory)\java\build\classes\java\test + if %errorlevel% neq 0 exit /b %errorlevel% + jar cvf $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\testing.jar . + if %errorlevel% neq 0 exit /b %errorlevel% + popd + pushd $(Build.SourcesDirectory)\java\build\resources\test + rd /s /q ai\onnxruntime\native + jar uvf $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\testing.jar . + popd + rd /s /q $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\stage + dir /s /b $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }} + workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo' + displayName: 'Add symbols and notices to Java' + + - task: PublishBuildArtifacts@1 + condition: and(succeeded(), eq('${{ parameters.buildJava}}', true)) + displayName: 'Publish Java temp binaries' + inputs: + pathtoPublish: '$(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}' + artifactName: 'drop-onnxruntime-java-win-${{ parameters.packageName }}' + + - ${{ if eq(parameters['DoCompliance'], 'true') }}: + - task: Semmle@0 + condition: and(succeeded(), eq('${{ parameters.buildJava}}', true)) + inputs: + sourceCodeDirectory: '$(Build.SourcesDirectory)\java' + language: 'java' + cleanupBuildCommands: 'C:\ProgramData\chocolatey\bin\gradle.exe -Dorg.gradle.daemon=false clean' + buildCommands: 'C:\ProgramData\chocolatey\bin\gradle.exe -Dorg.gradle.daemon=false jar' + querySuite: 'Recommended' + timeout: '7200' + ram: '16384' + addProjectDirToScanningExclusionList: true + + - task: PythonScript@0 + displayName: 'Regenerate cmake config with STATIC_ANALYSIS=ON ' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON' + workingDirectory: '$(Build.BinariesDirectory)' + + - template: compliance.yml + parameters : + msbuildPlatform: ${{ parameters.msbuildPlatform }} + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/win-esrp-dll.yml b/tools/ci_build/github/azure-pipelines/templates/win-esrp-dll.yml index ef4ab57300952..a51eb3f6a0b92 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-esrp-dll.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-esrp-dll.yml @@ -1,12 +1,24 @@ parameters: - FolderPath: '' - DisplayName: '' - DoEsrp: 'false' - Pattern: '*.dll' +- name: DoEsrp + type: boolean + default: true + +- name: FolderPath + type: string + default: '' + +- name: DisplayName + type: string + default: '' + +- name: Pattern + type: string + default: '*.dll' steps: - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: ${{ parameters.DisplayName }} + condition: and(succeeded(), eq('${{ parameters.DoEsrp }}', true)) inputs: ConnectedServiceName: 'OnnxRuntime CodeSign 20190817' FolderPath: ${{ parameters.FolderPath }} diff --git a/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml b/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml index 9202f43123dc5..e6e7cc3727e86 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml @@ -261,9 +261,13 @@ steps: searchPattern: '**/*.pdb' symbolServerType: teamServices - # Compliance tasks require logs from Debug Build - ${{ if eq(parameters['DoCompliance'], 'true') }}: - template: compliance.yml + parameters : + ${{ if ne(parameters.BuildArch, 'x86') }}: + msbuildPlatform: ${{ parameters.BuildArch }} + ${{ if eq(parameters.BuildArch, 'x86') }}: + msbuildPlatform: 'Win32' - template: component-governance-component-detection-steps.yml parameters : diff --git a/tools/ci_build/github/windows/bundle_dlls.bat b/tools/ci_build/github/windows/bundle_dlls.bat new file mode 100644 index 0000000000000..0c824a1f0f93f --- /dev/null +++ b/tools/ci_build/github/windows/bundle_dlls.bat @@ -0,0 +1,25 @@ +REM for available runtime identifiers, see https://github.com/dotnet/corefx/blob/release/3.1/pkg/Microsoft.NETCore.Platforms/runtime.json +set PATH=%CD%;%PATH% +SETLOCAL EnableDelayedExpansion +FOR /R %%i IN (*.nupkg) do ( + set filename=%%~ni + IF NOT "!filename:~25,7!"=="Managed" ( + mkdir runtimes\win-x86\native + mkdir runtimes\win10-arm\native + mkdir runtimes\win10-arm64\native + mkdir runtimes\linux-x64\native + mkdir runtimes\osx-x64\native + move onnxruntime-win-x86\lib\onnxruntime.dll runtimes\win-x86\native\onnxruntime.dll + move onnxruntime-win-x86\lib\onnxruntime.lib runtimes\win-x86\native\onnxruntime.lib + move onnxruntime-win-x86\lib\onnxruntime.pdb runtimes\win-x86\native\onnxruntime.pdb + move onnxruntime-win10-arm64\lib\onnxruntime.dll runtimes\win10-arm64\native\onnxruntime.dll + move onnxruntime-win10-arm64\lib\onnxruntime.lib runtimes\win10-arm64\native\onnxruntime.lib + move onnxruntime-win10-arm64\lib\onnxruntime.pdb runtimes\win10-arm64\native\onnxruntime.pdb + move onnxruntime-win10-arm\lib\onnxruntime.dll runtimes\win10-arm\native\onnxruntime.dll + move onnxruntime-win10-arm\lib\onnxruntime.lib runtimes\win10-arm\native\onnxruntime.lib + move onnxruntime-win10-arm\lib\onnxruntime.pdb runtimes\win10-arm\native\onnxruntime.pdb + move onnxruntime-linux-x64\lib\libonnxruntime.so.1* runtimes\linux-x64\native\libonnxruntime.so + move onnxruntime-osx-x64\lib\libonnxruntime.*.dylib runtimes\osx-x64\native\libonnxruntime.dylib + 7z a %%~ni.nupkg runtimes + ) +) \ No newline at end of file diff --git a/tools/ci_build/github/windows/extract_nuget_files.ps1 b/tools/ci_build/github/windows/extract_nuget_files.ps1 new file mode 100644 index 0000000000000..afb0c7db2cb66 --- /dev/null +++ b/tools/ci_build/github/windows/extract_nuget_files.ps1 @@ -0,0 +1,36 @@ +New-Item -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts -ItemType directory + +Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.zip | +Foreach-Object { + $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts" + Write-Output $cmd + Invoke-Expression -Command $cmd +} + +Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tgz | +Foreach-Object { + $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" + Write-Output $cmd + Invoke-Expression -Command $cmd +} + +Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tar | +Foreach-Object { + $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts" + Write-Output $cmd + Invoke-Expression -Command $cmd +} + + +New-Item -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\external\protobuf\cmake\RelWithDebInfo -ItemType directory + +Copy-Item -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-x64-*\lib\* -Destination $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo +Copy-Item -Path $Env:BUILD_BINARIESDIRECTORY\extra-artifact\protoc.exe $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\external\protobuf\cmake\RelWithDebInfo + +Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-arm-* | Rename-Item -NewName $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-arm +Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-arm64-* | Rename-Item -NewName $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-arm64 +Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-x86-* | Rename-Item -NewName $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-x86 + +Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-linux-x64-* | Rename-Item -NewName $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-linux-x64 + +Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-osx-x64-* | Rename-Item -NewName $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-osx-x64 \ No newline at end of file diff --git a/tools/ci_build/github/windows/jar_packaging.ps1 b/tools/ci_build/github/windows/jar_packaging.ps1 new file mode 100644 index 0000000000000..80ecd80b7f3b8 --- /dev/null +++ b/tools/ci_build/github/windows/jar_packaging.ps1 @@ -0,0 +1,14 @@ +$ErrorActionPreference = "Stop" +Write-Output "Start" +dir +pushd onnxruntime-java-linux-x64 +Write-Output "Run 7z" +7z a $Env:BUILD_BINARIESDIRECTORY\java-artifact\onnxruntime-java-win-x64\testing.jar libcustom_op_library.so +Remove-Item -Path libcustom_op_library.so +7z a $Env:BUILD_BINARIESDIRECTORY\java-artifact\onnxruntime-java-win-x64\onnxruntime-$Env:ONNXRUNTIMEVERSION.jar . +popd +pushd onnxruntime-java-osx-x64 +7z a $Env:BUILD_BINARIESDIRECTORY\java-artifact\onnxruntime-java-win-x64\testing.jar libcustom_op_library.dylib +Remove-Item -Path libcustom_op_library.dylib +7z a $Env:BUILD_BINARIESDIRECTORY\java-artifact\onnxruntime-java-win-x64\onnxruntime-$Env:ONNXRUNTIMEVERSION.jar . +popd \ No newline at end of file