From 40ede3d3fcc6ad28e1431dc819bac789d70de289 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 5 Feb 2025 11:33:10 -0800 Subject: [PATCH 1/8] Enable VCPKG in more places --- tools/ci_build/github/android/build_aar_package.py | 2 +- ...roid-arm64-v8a-QNN-crosscompile-ci-pipeline.yml | 2 +- .../android-x86_64-crosscompile-ci-pipeline.yml | 6 +++--- .../azure-pipelines/bigmodels-ci-pipeline.yml | 2 +- .../github/azure-pipelines/linux-ci-pipeline.yml | 6 +++--- .../azure-pipelines/linux-dnnl-ci-pipeline.yml | 2 +- .../azure-pipelines/linux-gpu-ci-pipeline.yml | 2 +- .../azure-pipelines/linux-migraphx-ci-pipeline.yml | 2 +- .../azure-pipelines/linux-qnn-ci-pipeline.yml | 4 ++-- .../azure-pipelines/linux-rocm-ci-pipeline.yml | 2 +- .../azure-pipelines/mac-coreml-ci-pipeline.yml | 2 +- .../nuget/templates/dml-vs-2022.yml | 2 +- .../stages/py-cpu-packaging-stage.yml | 2 +- .../azure-pipelines/stages/py-win-gpu-stage.yml | 2 +- .../azure-pipelines/templates/c-api-linux-cpu.yml | 2 +- .../templates/mac-cpu-packaging-steps.yml | 2 +- .../azure-pipelines/templates/py-win-arm64-qnn.yml | 2 +- .../templates/py-win-arm64ec-qnn.yml | 2 +- .../azure-pipelines/templates/py-win-x64-qnn.yml | 2 +- .../github/azure-pipelines/templates/rocm.yml | 2 +- .../github/azure-pipelines/templates/win-ci.yml | 14 +------------- 21 files changed, 26 insertions(+), 38 deletions(-) diff --git a/tools/ci_build/github/android/build_aar_package.py b/tools/ci_build/github/android/build_aar_package.py index c2bc5cba82a23..3f508dce3b84e 100644 --- a/tools/ci_build/github/android/build_aar_package.py +++ b/tools/ci_build/github/android/build_aar_package.py @@ -93,7 +93,7 @@ def _build_aar(args): aar_dir = os.path.join(intermediates_dir, "aar", build_config) jnilibs_dir = os.path.join(intermediates_dir, "jnilibs", build_config) exe_dir = os.path.join(intermediates_dir, "executables", build_config) - base_build_command = [sys.executable, BUILD_PY] + build_settings["build_params"] + ["--config=" + build_config] + base_build_command = [sys.executable, BUILD_PY] + build_settings["build_params"] + ["--config=" + build_config, "--use_vcpkg"] header_files_path = "" if qnn_android_build: diff --git a/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml index 9b7e887e66b81..2934e1e3e4d2e 100644 --- a/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml @@ -71,7 +71,7 @@ jobs: --android_ndk_path $ANDROID_NDK_HOME \ --android_abi=x86_64 \ --android_api=31 \ - --parallel \ + --parallel --use_vcpkg \ --build_shared_lib \ --use_qnn static_lib \ --qnn_home $(QnnSDKRootDir) \ diff --git a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml index a0ae0441f2824..e9da32f0d45f4 100644 --- a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml @@ -97,7 +97,7 @@ stages: --android_abi=x86_64 \ --android_api=30 \ --skip_submodule_sync \ - --parallel \ + --parallel --use_vcpkg \ --cmake_generator=Ninja \ --build_java displayName: CPU EP, Build and Test @@ -158,7 +158,7 @@ stages: --android_abi=x86_64 \ --android_api=29 \ --skip_submodule_sync \ - --parallel \ + --parallel --use_vcpkg \ --use_nnapi \ --build_shared_lib \ --cmake_generator=Ninja \ @@ -218,7 +218,7 @@ stages: --android_abi=x86_64 \ --android_api=29 \ --skip_submodule_sync \ - --parallel \ + --parallel --use_vcpkg \ --use_nnapi \ --build_shared_lib \ --cmake_generator=Ninja \ diff --git a/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml index b8710b6374093..78e1c53bf875d 100644 --- a/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml @@ -98,7 +98,7 @@ stages: --config Release --update --build \ --skip_submodule_sync \ --build_shared_lib \ - --parallel \ + --parallel --use_vcpkg \ --build_wheel \ --enable_onnx_tests --use_cuda --cuda_version=11.8 --cuda_home=/usr/local/cuda-11.8 --cudnn_home=/usr/local/cuda-11.8 \ --enable_cuda_profiling \ diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index 3874c98218038..979d82de64a13 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -83,7 +83,7 @@ stages: --config Debug \ --skip_submodule_sync \ --build_shared_lib \ - --parallel \ + --parallel --use_vcpkg \ --enable_onnx_tests --enable_address_sanitizer \ --update --build; python3 /onnxruntime_src/tools/ci_build/build.py \ @@ -91,7 +91,7 @@ stages: --config Debug \ --skip_submodule_sync \ --build_shared_lib \ - --parallel \ + --parallel --use_vcpkg \ --enable_onnx_tests --enable_address_sanitizer \ --test; ' @@ -199,7 +199,7 @@ stages: --config Release \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_binskim_compliant_compile_flags \ + --parallel --use_vcpkg --use_binskim_compliant_compile_flags \ --build_wheel \ --build_csharp \ --enable_onnx_tests \ diff --git a/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml index b9940045c009e..7391f99c2d333 100644 --- a/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml @@ -67,7 +67,7 @@ jobs: --config Debug Release \ --skip_submodule_sync \ --build_shared_lib \ - --parallel \ + --parallel --use_vcpkg \ --enable_pybind \ --enable_onnx_tests \ --build_java \ diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index 71f7ab6e49b70..91c33820ac852 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -209,7 +209,7 @@ stages: cd /onnxruntime_src/java && /onnxruntime_src/java/gradlew cmakeCheck -DcmakeBuildDir=/build/Release -DUSE_CUDA=1; \ cd /tmp; \ python3 /onnxruntime_src/tools/ci_build/build.py \ - --build_dir /build --config Release --test --skip_submodule_sync --build_shared_lib --parallel --use_binskim_compliant_compile_flags --build_wheel --enable_onnx_tests \ + --build_dir /build --config Release --test --skip_submodule_sync --build_shared_lib --parallel --use_vcpkg --use_binskim_compliant_compile_flags --build_wheel --enable_onnx_tests \ --enable_transformers_tool_test --use_cuda --cuda_version=${{parameters.CudaVersion}} --cuda_home=/usr/local/cuda --cudnn_home=/usr/local/cuda \ --enable_pybind --build_java --ctest_path "" ; \ ' diff --git a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml index c6efd306ed43f..d44b6a1a9ce21 100644 --- a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml @@ -111,7 +111,7 @@ jobs: --update \ --build_dir /build \ --build \ - --parallel \ + --parallel --use_vcpkg \ --build_wheel \ --skip_submodule_sync \ --use_cache \ diff --git a/tools/ci_build/github/azure-pipelines/linux-qnn-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-qnn-ci-pipeline.yml index 4c3c504eb00ef..41eb559831e69 100644 --- a/tools/ci_build/github/azure-pipelines/linux-qnn-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-qnn-ci-pipeline.yml @@ -74,7 +74,7 @@ jobs: --use_qnn $(QnnLibKind) \ --qnn_home $(QnnSDKRootDir) \ --cmake_generator=Ninja \ - --update --build --parallel + --update --build --parallel --use_vcpkg displayName: Build QNN EP - script: | @@ -87,7 +87,7 @@ jobs: --use_qnn $(QnnLibKind) \ --qnn_home $(QnnSDKRootDir) \ --cmake_generator=Ninja \ - --test + --test --use_vcpkg displayName: Run unit tests - task: CmdLine@2 diff --git a/tools/ci_build/github/azure-pipelines/linux-rocm-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-rocm-ci-pipeline.yml index 19390895aa479..af52d7b883066 100644 --- a/tools/ci_build/github/azure-pipelines/linux-rocm-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-rocm-ci-pipeline.yml @@ -197,7 +197,7 @@ jobs: --enable_nccl \ --build_dir /build \ --build_shared_lib \ - --parallel \ + --parallel --use_vcpkg \ --build_wheel \ --skip_submodule_sync \ --test --enable_onnx_tests --enable_transformers_tool_test \ diff --git a/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml index a1059bb3c314e..9549d9e0604b0 100644 --- a/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml @@ -58,7 +58,7 @@ jobs: --build_dir build \ --skip_submodule_sync \ --cmake_generator=Ninja \ - --parallel --use_binskim_compliant_compile_flags \ + --parallel --use_vcpkg --use_binskim_compliant_compile_flags \ --build_shared_lib \ --config Debug \ --use_cache \ diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml index 7d43c470ade8b..cf1a2809081c9 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml @@ -98,7 +98,7 @@ stages: - ${{ else }}: - powershell: | - python tools\ci_build\build.py ${{ parameters.BuildCommand }} --use_binskim_compliant_compile_flags --parallel --build_csharp --build --update --config $(BuildConfig) --build_nuget --msbuild_extra_options IncludeMobileTargets=false ${{ variables.build_py_lto_flag }} + python tools\ci_build\build.py ${{ parameters.BuildCommand }} --use_binskim_compliant_compile_flags --parallel --use_vcpkg --build_csharp --build --update --config $(BuildConfig) --build_nuget --msbuild_extra_options IncludeMobileTargets=false ${{ variables.build_py_lto_flag }} - ${{ if notIn(parameters['sln_platform'], 'Win32', 'x64') }}: # Use cross-compiled protoc diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml index fa4561f082c37..e93ac975809d3 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml @@ -229,7 +229,7 @@ stages: set -e -x export _PYTHON_HOST_PLATFORM=macosx-${{variables.MACOSX_DEPLOYMENT_TARGET}}-universal2 python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --use_coreml --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --config Release --build_wheel ${{ parameters.build_py_parameters }} --use_coreml --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64" --update --build + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --use_coreml --skip_submodule_sync --parallel --use_vcpkg --use_binskim_compliant_compile_flags --config Release --build_wheel ${{ parameters.build_py_parameters }} --use_coreml --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64" --update --build displayName: 'Command Line Script' - script: | diff --git a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml index ce098b490f1b5..28a9b1824c769 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml @@ -125,7 +125,7 @@ stages: --cmake_generator "$(VSGenerator)" --enable_pybind --enable_onnx_tests - --parallel 8 --use_binskim_compliant_compile_flags --update --build --msvc_toolset 14.40 + --parallel 8 --use_vcpkg --use_binskim_compliant_compile_flags --update --build --msvc_toolset 14.40 $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} ${{ variables.trt_build_flag }} workingDirectory: '$(Build.BinariesDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml index 891f821589042..12d53c1a1b8e7 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml @@ -66,7 +66,7 @@ jobs: docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging /bin/bash -c "python3 \ /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \ - --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --use_vcpkg --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/installed" + --skip_submodule_sync --parallel --use_vcpkg --use_binskim_compliant_compile_flags --use_vcpkg --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/installed" mkdir $(Build.ArtifactStagingDirectory)/testdata cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata ls -al $(Build.ArtifactStagingDirectory) diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml index c9f41ecc143ab..3d61990ead0c2 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml @@ -48,7 +48,7 @@ steps: - script: | set -e -x rm -rf $(Build.BinariesDirectory)/Release - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --update --build ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --config Release --use_vcpkg + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --update --build ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --use_vcpkg --use_binskim_compliant_compile_flags --build_shared_lib --config Release --use_vcpkg cd $(Build.BinariesDirectory)/Release make install DESTDIR=$(Build.BinariesDirectory)/installed displayName: 'Build ${{ parameters.MacosArch }}' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml index e22b5f1f1357e..e61e4dac99f28 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml @@ -98,7 +98,7 @@ jobs: --use_qnn --qnn_home $(QnnSDKRootDir) --enable_pybind - --parallel --update + --parallel --use_vcpkg --update $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} workingDirectory: '$(Build.BinariesDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml index 5d8ac8b8d7228..ef52ffd0cc097 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml @@ -83,7 +83,7 @@ jobs: --use_qnn --qnn_home $(QnnSDKRootDir) --enable_pybind - --parallel --update --arm64ec + --parallel --use_vcpkg --update --arm64ec $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} workingDirectory: '$(Build.BinariesDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml index 6f96b7b6a0795..e6b13bf22b9e4 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml @@ -76,7 +76,7 @@ jobs: --use_qnn --qnn_home $(QnnSDKRootDir) --enable_pybind - --parallel --update + --parallel --use_vcpkg --update $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} workingDirectory: '$(Build.BinariesDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/templates/rocm.yml b/tools/ci_build/github/azure-pipelines/templates/rocm.yml index 47c0ba3eb2d1e..25517ada74ba3 100644 --- a/tools/ci_build/github/azure-pipelines/templates/rocm.yml +++ b/tools/ci_build/github/azure-pipelines/templates/rocm.yml @@ -89,7 +89,7 @@ jobs: --rocm_home=/opt/rocm \ --nccl_home=/opt/rocm \ --update \ - --parallel \ + --parallel --use_vcpkg \ --build_dir /build \ --build \ --build_wheel \ diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 61c28e0ae27bc..1bda2cc1432c4 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -161,22 +161,10 @@ stages: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} $(timeoutParameter) $(buildJavaParameter)' + arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --build --use_vcpkg --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} $(timeoutParameter) $(buildJavaParameter)' workingDirectory: '$(Build.BinariesDirectory)' - - task: VSBuild@1 - displayName: 'Build' - inputs: - solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' - platform: ${{ parameters.msbuildPlatform }} - configuration: RelWithDebInfo - msbuildArchitecture: ${{ parameters.buildArch }} - maximumCpuCount: true # default is num logical cores worth of projects building concurrently - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - createLogFile: true - # For CPU job, tests are run in the same machine as building - ${{ if eq(parameters.buildJava, 'true') }}: - template: make_java_win_binaries.yml From ed91a326989bc6ee39aaa6822d7ac74979c98eb7 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 5 Feb 2025 13:16:34 -0800 Subject: [PATCH 2/8] update --- cmake/external/onnxruntime_external_deps.cmake | 6 +++--- cmake/onnxruntime_framework.cmake | 7 ++----- cmake/onnxruntime_providers_cpu.cmake | 3 +-- cmake/onnxruntime_python.cmake | 2 +- tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml | 2 +- .../linux/docker/migraphx-ci-pipeline-env.Dockerfile | 6 +++--- 6 files changed, 11 insertions(+), 15 deletions(-) diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake index e4c3e511bdd33..442737d844c24 100644 --- a/cmake/external/onnxruntime_external_deps.cmake +++ b/cmake/external/onnxruntime_external_deps.cmake @@ -600,6 +600,7 @@ endif() if(onnxruntime_ENABLE_DLPACK) message(STATUS "dlpack is enabled.") + onnxruntime_fetchcontent_declare( dlpack URL ${DEP_URL_dlpack} @@ -607,9 +608,8 @@ if(onnxruntime_ENABLE_DLPACK) EXCLUDE_FROM_ALL FIND_PACKAGE_ARGS NAMES dlpack ) - # We can't use onnxruntime_fetchcontent_makeavailable since some part of the the dlpack code is Linux only. - # For example, dlpackcpp.h uses posix_memalign. - FetchContent_Populate(dlpack) + onnxruntime_fetchcontent_makeavailable(dlpack) + endif() endif() if(onnxruntime_ENABLE_TRAINING OR (onnxruntime_ENABLE_TRAINING_APIS AND onnxruntime_BUILD_UNIT_TESTS)) diff --git a/cmake/onnxruntime_framework.cmake b/cmake/onnxruntime_framework.cmake index 969d604c88c50..cfed6c4e96089 100644 --- a/cmake/onnxruntime_framework.cmake +++ b/cmake/onnxruntime_framework.cmake @@ -77,8 +77,7 @@ endif() if (onnxruntime_ENABLE_TRAINING_OPS) target_include_directories(onnxruntime_framework PRIVATE ${ORTTRAINING_ROOT}) if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP OR onnxruntime_ENABLE_TRITON) - onnxruntime_add_include_to_target(onnxruntime_framework Python::Module) - target_include_directories(onnxruntime_framework PRIVATE ${dlpack_SOURCE_DIR}/include) + onnxruntime_add_include_to_target(onnxruntime_framework Python::Module dlpack::dlpack) endif() endif() if (onnxruntime_USE_MPI) @@ -86,9 +85,7 @@ if (onnxruntime_USE_MPI) endif() if (onnxruntime_ENABLE_ATEN) - # DLPack is a header-only dependency - set(DLPACK_INCLUDE_DIR ${dlpack_SOURCE_DIR}/include) - target_include_directories(onnxruntime_framework PRIVATE ${DLPACK_INCLUDE_DIR}) + onnxruntime_add_include_to_target(onnxruntime_framework dlpack::dlpack) endif() onnxruntime_add_include_to_target(onnxruntime_framework onnxruntime_common onnx onnx_proto ${PROTOBUF_LIB} flatbuffers::flatbuffers safeint_interface Boost::mp11 nlohmann_json::nlohmann_json) diff --git a/cmake/onnxruntime_providers_cpu.cmake b/cmake/onnxruntime_providers_cpu.cmake index 3e1a9edbd13ba..5d2f01046dec5 100644 --- a/cmake/onnxruntime_providers_cpu.cmake +++ b/cmake/onnxruntime_providers_cpu.cmake @@ -196,8 +196,7 @@ endif() if (onnxruntime_ENABLE_DLPACK) target_compile_definitions(onnxruntime_providers PRIVATE ENABLE_DLPACK) # DLPack is a header-only dependency - set(DLPACK_INCLUDE_DIR ${dlpack_SOURCE_DIR}/include) - target_include_directories(onnxruntime_providers PRIVATE ${DLPACK_INCLUDE_DIR}) + onnxruntime_add_include_to_target(onnxruntime_providers dlpack::dlpack) endif() if (onnxruntime_ENABLE_TRAINING) diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index 7d7416cc9115e..0ae3b338db12f 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -131,7 +131,7 @@ if (onnxruntime_ENABLE_ATEN) endif() if (onnxruntime_ENABLE_DLPACK) - target_include_directories(onnxruntime_pybind11_state PRIVATE ${dlpack_SOURCE_DIR}/include) + onnxruntime_add_include_to_target(onnxruntime_providers dlpack::dlpack) endif() if (onnxruntime_ENABLE_TRAINING) diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index 7c0142d041391..f3408d9168807 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -106,7 +106,7 @@ stages: parameters: BuildConfig: 'RelWithDebInfo' buildArch: x64 - additionalBuildFlags: --build_wheel --use_dnnl + additionalBuildFlags: --build_wheel --use_dnnl --use_vcpkg msbuildPlatform: x64 isX86: false job_name_suffix: x64_release diff --git a/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile b/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile index 51591e11ea2e9..e594c6891eee8 100644 --- a/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile +++ b/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile @@ -12,7 +12,7 @@ RUN echo "$APT_PREF" > /etc/apt/preferences.d/rocm-pin-600 ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates curl libnuma-dev gnupg && \ + apt-get install -y --no-install-recommends ca-certificates ninja-build git zip curl libnuma-dev gnupg && \ curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - &&\ printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ jammy main" | tee /etc/apt/sources.list.d/rocm.list && \ printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu jammy main" | tee /etc/apt/sources.list.d/amdgpu.list && \ @@ -45,10 +45,10 @@ ENV LANG C.UTF-8 WORKDIR /stage # Cmake -ENV CMAKE_VERSION=3.30.1 +ENV CMAKE_VERSION=3.31.5 RUN cd /usr/local && \ wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz && \ - tar -zxf /usr/local/cmake-3.30.1-Linux-x86_64.tar.gz --strip=1 -C /usr + tar -zxf /usr/local/cmake-3.31.5-Linux-x86_64.tar.gz --strip=1 -C /usr # ccache RUN mkdir -p /tmp/ccache && \ From 9d321392bd15765c8fc4f6ed8e71932f88bd5e2a Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 5 Feb 2025 21:29:09 +0000 Subject: [PATCH 3/8] fix --- cmake/external/onnxruntime_external_deps.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake index 442737d844c24..1e09145baa913 100644 --- a/cmake/external/onnxruntime_external_deps.cmake +++ b/cmake/external/onnxruntime_external_deps.cmake @@ -609,7 +609,6 @@ if(onnxruntime_ENABLE_DLPACK) FIND_PACKAGE_ARGS NAMES dlpack ) onnxruntime_fetchcontent_makeavailable(dlpack) - endif() endif() if(onnxruntime_ENABLE_TRAINING OR (onnxruntime_ENABLE_TRAINING_APIS AND onnxruntime_BUILD_UNIT_TESTS)) From 5a89d405d29a5836797c160fe86d65d35e46a238 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 5 Feb 2025 21:32:34 +0000 Subject: [PATCH 4/8] update --- cmake/onnxruntime_python.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index 0ae3b338db12f..f1a685cd88b28 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -131,7 +131,7 @@ if (onnxruntime_ENABLE_ATEN) endif() if (onnxruntime_ENABLE_DLPACK) - onnxruntime_add_include_to_target(onnxruntime_providers dlpack::dlpack) + target_link_libraries(onnxruntime_providers PRIVATE dlpack::dlpack) endif() if (onnxruntime_ENABLE_TRAINING) From 83c0af13f6e8d9fdf1b3c7351590b142d50faf56 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 5 Feb 2025 21:41:37 +0000 Subject: [PATCH 5/8] update --- cmake/onnxruntime_python.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index f1a685cd88b28..aee6d2ff7655c 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -131,7 +131,7 @@ if (onnxruntime_ENABLE_ATEN) endif() if (onnxruntime_ENABLE_DLPACK) - target_link_libraries(onnxruntime_providers PRIVATE dlpack::dlpack) + target_link_libraries(onnxruntime_pybind11_state PRIVATE dlpack::dlpack) endif() if (onnxruntime_ENABLE_TRAINING) From 4d6abed3faaba79264630ac1ad01a8c12aba4d9f Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 5 Feb 2025 21:52:14 +0000 Subject: [PATCH 6/8] format code --- tools/ci_build/github/android/build_aar_package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/github/android/build_aar_package.py b/tools/ci_build/github/android/build_aar_package.py index 3f508dce3b84e..de43dc9528ad0 100644 --- a/tools/ci_build/github/android/build_aar_package.py +++ b/tools/ci_build/github/android/build_aar_package.py @@ -93,7 +93,9 @@ def _build_aar(args): aar_dir = os.path.join(intermediates_dir, "aar", build_config) jnilibs_dir = os.path.join(intermediates_dir, "jnilibs", build_config) exe_dir = os.path.join(intermediates_dir, "executables", build_config) - base_build_command = [sys.executable, BUILD_PY] + build_settings["build_params"] + ["--config=" + build_config, "--use_vcpkg"] + base_build_command = ( + [sys.executable, BUILD_PY] + build_settings["build_params"] + ["--config=" + build_config, "--use_vcpkg"] + ) header_files_path = "" if qnn_android_build: From b94dcfbd68c28f87e24469af5d5f2a2047e6a545 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 5 Feb 2025 22:21:08 +0000 Subject: [PATCH 7/8] update --- .../github/azure-pipelines/linux-migraphx-ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml index d44b6a1a9ce21..95171998c7839 100644 --- a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml @@ -111,7 +111,7 @@ jobs: --update \ --build_dir /build \ --build \ - --parallel --use_vcpkg \ + --parallel --use_vcpkg --cmake_generator=Ninja \ --build_wheel \ --skip_submodule_sync \ --use_cache \ From 806767226375ebe4517aebdefe042fdae3140fe7 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 5 Feb 2025 23:53:20 +0000 Subject: [PATCH 8/8] revert --- .../github/azure-pipelines/linux-migraphx-ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml index 95171998c7839..c6efd306ed43f 100644 --- a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml @@ -111,7 +111,7 @@ jobs: --update \ --build_dir /build \ --build \ - --parallel --use_vcpkg --cmake_generator=Ninja \ + --parallel \ --build_wheel \ --skip_submodule_sync \ --use_cache \