From b0497c71245d20bc61c91ba382a9c50cb01e2e76 Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Wed, 5 Mar 2025 15:35:57 -0800 Subject: [PATCH] Create a packaging pipeline for a custom nuget package --- .../custom-nuget-packaging-pipeline.yml | 142 +++++++++++++++++ ...acts-package-and-publish-steps-windows.yml | 16 ++ .../azure-pipelines/templates/qnn-ep-win.yml | 27 +++- .../azure-pipelines/templates/win-ci.yml | 2 +- .../nuget/generate_nuspec_for_custom_nuget.py | 150 ++++++++++++++++++ 5 files changed, 331 insertions(+), 6 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/custom-nuget-packaging-pipeline.yml create mode 100644 tools/nuget/generate_nuspec_for_custom_nuget.py diff --git a/tools/ci_build/github/azure-pipelines/custom-nuget-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/custom-nuget-packaging-pipeline.yml new file mode 100644 index 0000000000000..8aaaa0e85585a --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/custom-nuget-packaging-pipeline.yml @@ -0,0 +1,142 @@ +parameters: +- name: CudaVersion + type: string + default: '12.2' + +- name: QnnSdk + displayName: QNN SDK Version + type: string + default: 2.31.0.250130 + +- name: IsReleaseBuild + displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release. + type: boolean + default: false + +- name: PackageName + displayName: What is the package name? + type: string + default: 'Microsoft.ML.OnnxRuntime.Flamingo' + +variables: + - template: templates/common-variables.yml + - name: ReleaseVersionSuffix + value: '' + - name: win_cuda_home + ${{ if eq(parameters.CudaVersion, '11.8') }}: + value: $(Agent.TempDirectory)\v11.8 + ${{ if eq(parameters.CudaVersion, '12.2') }}: + value: $(Agent.TempDirectory)\v12.2 + +stages: + - template: templates/win-ci.yml + parameters: + ort_build_pool_name: 'onnxruntime-Win2022-GPU-A10' + DoCompliance: false + DoEsrp: true + stage_name_suffix: CUDA + buildArch: x64 + msbuildPlatform: x64 + packageName: x64-cuda + CudaVersion: ${{ parameters.CudaVersion }} + buildparameter: --use_cuda --cuda_home=${{ variables.win_cuda_home }} --enable_onnx_tests --enable_wcos --use_webgpu --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52-real;61-real;75-real;86-real;89-real;90-virtual" + runTests: false + buildJava: false + java_artifact_id: onnxruntime_gpu + UseIncreasedTimeoutForTests: false + SpecificArtifact: false + BuildId: '0' + + - template: templates/qnn-ep-win.yml + parameters: + qnn_ep_build_pool_name: 'Onnxruntime-QNNEP-Windows-2022-CPU' + QnnSdk: ${{ parameters.QnnSdk }} + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + DoEsrp: true + ArtifactName: 'drop-nuget-qnn-arm64' + # Add --use_webgpu to enable WebGPU + buildParameter: '--arm64' + buildPlatform: 'ARM64' + buildArch: 'ARM64' + StageName: 'OnnxRuntime_QNN_Nuget_Win_Arm64' + build_config: 'RelWithDebInfo' + Is1ES: false + PublishArchive: true + + - stage: NugetPackaging + dependsOn: [Windows_Packaging_CUDA, OnnxRuntime_QNN_Nuget_Win_Arm64] + jobs: + - job: CreateNugetPackage + pool: 'Onnxruntime-Win2022-GPU-A10' + timeoutInMinutes: 120 + steps: + - checkout: self + clean: true + submodules: none + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + addToPath: true + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - managed nuget' + inputs: + artifactName: 'drop-nuget-qnn-arm64' + targetPath: '$(Build.BinariesDirectory)/managed-nuget' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - win-x64' + inputs: + artifactName: 'onnxruntime-win-x64-cuda' + targetPath: '$(Build.BinariesDirectory)/win-x64' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - win-arm64' + inputs: + artifactName: 'onnxruntime-win-ARM64-qnn' + targetPath: '$(Build.BinariesDirectory)/win-arm64' + + - task: PowerShell@2 + displayName: 'Extract Nuget Package Version' + inputs: + targetType: 'inline' + script: | + $nupkgs = (Get-ChildItem $(Build.BinariesDirectory)/managed-nuget -Filter Microsoft.ML.OnnxRuntime.Managed.*.nupkg -Recurse) + $package_name = $nupkgs[0].Name + $version_length = $package_name.Length - "Microsoft.ML.OnnxRuntime.Managed.".Length - ".nupkg".Length + $package_version = $package_name.Substring("Microsoft.ML.OnnxRuntime.Managed.".Length, $version_length) + Write-Host "##vso[task.setvariable variable=package_version;]$package_version" + workingDirectory: $(Build.BinariesDirectory) + + - task: PowerShell@2 + displayName: 'Extract Archives' + inputs: + targetType: 'inline' + script: | + Expand-Archive -Path $(Build.BinariesDirectory)/win-x64/onnxruntime-win-x64-cuda*.zip -DestinationPath $(Build.BinariesDirectory)/win-x64 + Expand-Archive -Path $(Build.BinariesDirectory)/win-arm64/onnxruntime-win-ARM64-qnn*.zip -DestinationPath $(Build.BinariesDirectory)/win-arm64 + $win_x64 = (Get-ChildItem -Path $(Build.BinariesDirectory)/win-x64 -Filter onnxruntime-win-x64-cuda*)[0].FullName + $win_arm64 = (Get-ChildItem -Path $(Build.BinariesDirectory)/win-arm64 -Filter onnxruntime-win-ARM64-qnn*)[0].FullName + Write-Host "##vso[task.setvariable variable=win_x64;]$win_x64" + Write-Host "##vso[task.setvariable variable=win_arm64;]$win_arm64" + workingDirectory: $(Build.BinariesDirectory) + + - task: PythonScript@0 + displayName: 'Generate Nuget Package' + inputs: + scriptPath: '$(Build.SourcesDirectory)/tools/nuget/generate_nuspec_for_custom_nuget.py' + arguments: '--nuspec_path "$(Build.BinariesDirectory)/${{ parameters.PackageName }}.nuspec" --root_dir "$(Build.SourcesDirectory)" --commit_id "$(Build.SourceVersion)" --win_arm64 "$(win_arm64)" --win_x64 "$(win_x64)" --package_version "$(package_version)" --package_name "${{ parameters.PackageName }}"' + + - task: NuGetCommand@2 + displayName: 'Pack Nuget Package' + inputs: + command: 'pack' + packagesToPack: '$(Build.BinariesDirectory)/${{ parameters.PackageName }}.nuspec' + packDestination: $(Build.ArtifactStagingDirectory)\ + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: Nuget' + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: '${{ parameters.PackageName }}' diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml index 5ee425405ac70..e1a514ea54123 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml @@ -57,6 +57,22 @@ steps: copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\onnxruntime_providers_cuda.pdb $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\onnxruntime_providers_cuda.lib $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib + # Copy WebGPU dependencies if required + copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\dxcompiler.dll $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib + copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\dxil.dll $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib + + # Copy QNN dependencies if required + copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\onnxruntime_providers_qnn.dll $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib + copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\libQnnHtp*.so $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib /Y + copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\libqnnhtp*.cat $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib /Y + copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\QnnCpu.dll $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib + copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\QnnHtp.dll $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib + copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\QnnHtpPrepare.dll $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib + copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\QnnHtpV68Stub.dll $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib + copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\QnnHtpV73Stub.dll $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib + copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\QnnSaver.dll $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib + copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\QnnSystem.dll $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib + # copy trt ep libraries only when trt ep is enabled copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\onnxruntime_providers_tensorrt.dll $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\onnxruntime_providers_tensorrt.pdb $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib diff --git a/tools/ci_build/github/azure-pipelines/templates/qnn-ep-win.yml b/tools/ci_build/github/azure-pipelines/templates/qnn-ep-win.yml index b591a3e3e121b..3fa4799ec9c0e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/qnn-ep-win.yml +++ b/tools/ci_build/github/azure-pipelines/templates/qnn-ep-win.yml @@ -10,6 +10,8 @@ parameters: buildPlatform: 'x64' buildArch: 'x64' StageName: 'OnnxRuntime_QNN_Nuget_Win_x64' + Is1ES: true + PublishArchive: false stages: - stage: ${{ parameters.StageName }} @@ -107,6 +109,14 @@ stages: DoEsrp: ${{ parameters.DoEsrp }} Pattern: 'onnxruntime*.dll' + - ${{ if eq(parameters.PublishArchive, true) }}: + - template: c-api-artifacts-package-and-publish-steps-windows.yml + parameters: + buildConfig: ${{ parameters.build_config }} + artifactName: 'onnxruntime-win-${{ parameters.buildPlatform }}-qnn' + artifactNameNoVersionString: 'onnxruntime-win-${{ parameters.buildPlatform }}-qnn' + DoEsrp: ${{ parameters.DoEsrp }} + - task: MSBuild@1 displayName: 'Restore NuGet Packages and create project.assets.json' inputs: @@ -155,8 +165,15 @@ stages: Contents: '*.snupkg' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: 1ES.PublishPipelineArtifact@1 - displayName: 'Publish Pipeline x64 NuGet Artifact' - inputs: - artifactName: ${{ parameters.ArtifactName }} - targetPath: '$(Build.ArtifactStagingDirectory)' + - ${{ if eq(parameters.Is1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Pipeline x64 NuGet Artifact' + inputs: + artifactName: ${{ parameters.ArtifactName }} + targetPath: '$(Build.ArtifactStagingDirectory)' + - ${{ else }}: + - task: PublishPipelineArtifact@1 + displayName: 'Publish Pipeline x64 NuGet Artifact' + inputs: + artifactName: ${{ parameters.ArtifactName }} + targetPath: '$(Build.ArtifactStagingDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 600e6d857185f..69a06c3db24b8 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -161,7 +161,7 @@ stages: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --build --use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} $(timeoutParameter) $(buildJavaParameter)' + arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --build --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} $(timeoutParameter) $(buildJavaParameter)' workingDirectory: '$(Build.BinariesDirectory)' diff --git a/tools/nuget/generate_nuspec_for_custom_nuget.py b/tools/nuget/generate_nuspec_for_custom_nuget.py new file mode 100644 index 0000000000000..baf46743cbf1b --- /dev/null +++ b/tools/nuget/generate_nuspec_for_custom_nuget.py @@ -0,0 +1,150 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import glob +import os +import shutil + +from generate_nuspec_for_native_nuget import generate_metadata + + +def generate_files(lines, args): + files_list = [""] + platform_map = { + "win-arm64": args.win_arm64, + "win-x64": args.win_x64, + } + + avoid_keywords = {"pdb"} + processed_includes = set() + for platform, platform_dir in platform_map.items(): + for file in glob.glob(os.path.join(platform_dir, "lib", "*")): + if not os.path.isfile(file): + continue + if any(keyword in file for keyword in avoid_keywords): + continue + file_name = os.path.basename(file) + + files_list.append(f'') + + for file in glob.glob(os.path.join(platform_dir, "include", "*")): + if not os.path.isfile(file): + continue + file_name = os.path.basename(file) + if file_name in processed_includes: + continue + processed_includes.add(file_name) + files_list.append(f'') + + files_list.append( + f'' + ) + + files_list.append(f'') + files_list.append( + f'' + ) + files_list.append(f'') + files_list.append( + f'' + ) + + source_props = os.path.join( + args.root_dir, + "csharp", + "src", + "Microsoft.ML.OnnxRuntime", + "targets", + "netstandard", + "props.xml", + ) + target_props = os.path.join( + args.root_dir, + "csharp", + "src", + "Microsoft.ML.OnnxRuntime", + "targets", + "netstandard", + f"{args.package_name}.props", + ) + shutil.copyfile(source_props, target_props) + files_list.append(f'') + files_list.append(f'') + + source_targets = os.path.join( + args.root_dir, + "csharp", + "src", + "Microsoft.ML.OnnxRuntime", + "targets", + "netstandard", + "targets.xml", + ) + target_targets = os.path.join( + args.root_dir, + "csharp", + "src", + "Microsoft.ML.OnnxRuntime", + "targets", + "netstandard", + f"{args.package_name}.targets", + ) + shutil.copyfile(source_targets, target_targets) + files_list.append(f'') + files_list.append(f'') + + files_list.append("") + lines.extend(files_list) + + +def parse_arguments(): + parser = argparse.ArgumentParser( + description="Create a nuspec file for the custom nuget package.", + ) + + parser.add_argument("--nuspec_path", required=True, help="Nuspec output file path.") + parser.add_argument("--root_dir", required=True, help="ORT repository root.") + parser.add_argument( + "--commit_id", + required=True, + help="The last commit id included in this package.", + ) + parser.add_argument("--win_arm64", required=True, help="Ort win-arm64 directory") + parser.add_argument("--win_x64", required=True, help="Ort win-x64 directory") + parser.add_argument("--package_version", required=True, help="Version of the package") + parser.add_argument("--package_name", required=True, help="Name of the package") + + args = parser.parse_args() + + args.sdk_info = "" + + return args + + +def generate_nuspec(args: argparse.Namespace): + lines = [''] + lines.append("") + + generate_metadata(lines, args) + generate_files(lines, args) + + lines.append("") + return lines + + +def main(): + args = parse_arguments() + + lines = generate_nuspec(args) + + with open(os.path.join(args.nuspec_path), "w") as f: + for line in lines: + # Uncomment the printing of the line if you need to debug what's produced on a CI machine + print(line) + f.write(line) + f.write("\n") + + +if __name__ == "__main__": + main()