From 086834ff72cb5fa8be93e5c1dbd58ed167259856 Mon Sep 17 00:00:00 2001 From: Erik Date: Wed, 15 Apr 2026 16:58:21 -0400 Subject: [PATCH 1/3] Update pipeline --- .../azure-pipelines/py-packaging-pipeline.yml | 30 --- .../stages/py-cpu-packaging-stage.yml | 26 --- .../setup-feeds-and-python-steps.yml | 189 +++++++++--------- 3 files changed, 99 insertions(+), 146 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml index 2fc3f30bd04c7..3211707ccf53a 100644 --- a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml @@ -19,26 +19,6 @@ parameters: type: boolean default: true -- name: enable_windows_arm64_qnn - displayName: 'Whether Windows ARM64 package with QNN EP is built.' - type: boolean - default: true - -- name: enable_windows_arm64ec_qnn - displayName: 'Whether Windows ARM64EC package with QNN EP is built.' - type: boolean - default: true - -- name: enable_windows_x64_qnn - displayName: 'Whether Windows x86_64 package with QNN EP is built.' - type: boolean - default: true - -- name: enable_linux_x64_qnn - displayName: 'Whether Linux x86_64 package with QNN EP is built.' - type: boolean - default: true - - name: build_py_parameters displayName: 'Specify extra build parameters' type: string @@ -55,11 +35,6 @@ parameters: - RelWithDebInfo - MinSizeRel -# Only applies to QNN packages. -- name: qnn_sdk_version - type: string - displayName: 'QNN SDK version. Only for QNN packages.' - default: 2.42.0.251225 trigger: none @@ -103,10 +78,5 @@ extends: enable_windows_cpu: ${{ parameters.enable_windows_cpu }} enable_mac_cpu: ${{ parameters.enable_mac_cpu }} enable_linux_arm: ${{ parameters.enable_linux_arm }} - enable_windows_arm64_qnn: ${{ parameters.enable_windows_arm64_qnn }} - enable_windows_arm64ec_qnn: ${{ parameters.enable_windows_arm64ec_qnn }} - enable_windows_x64_qnn: ${{ parameters.enable_windows_x64_qnn }} - enable_linux_x64_qnn: ${{ parameters.enable_linux_x64_qnn }} build_py_parameters: ${{ parameters.build_py_parameters }} cmake_build_type: ${{ parameters.cmake_build_type }} - qnn_sdk_version: ${{ parameters.qnn_sdk_version }} diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml index 88d2981e2ccaa..6beff17adc31c 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml @@ -25,26 +25,6 @@ parameters: type: boolean default: true -- name: enable_windows_arm64_qnn - displayName: 'Whether Windows ARM64 package with QNN EP is built.' - type: boolean - default: true - -- name: enable_windows_arm64ec_qnn - displayName: 'Whether Windows ARM64EC package with QNN EP is built.' - type: boolean - default: true - -- name: enable_windows_x64_qnn - displayName: 'Whether Windows x86_64 package with QNN EP is built.' - type: boolean - default: true - -- name: enable_linux_x64_qnn - displayName: 'Whether Linux x86_64 package with QNN EP is built.' - type: boolean - default: true - - name: cmake_build_type type: string displayName: 'Linux packages cmake build type. Linux Only.' @@ -55,12 +35,6 @@ parameters: - RelWithDebInfo - MinSizeRel -# Only applies to QNN packages. -- name: qnn_sdk_version - type: string - displayName: 'QNN SDK version. Only for QNN packages.' - default: 2.42.0.251225 - stages: - ${{ if eq(parameters.enable_windows_cpu, true) }}: - stage: Python_Packaging_Windows_CPU diff --git a/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml b/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml index c074270ce5949..ab424f397f0f5 100644 --- a/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml @@ -48,24 +48,27 @@ steps: ${{ if ne(parameters.architecture, '') }}: architecture: ${{ parameters.architecture }} -- pwsh: | - # Check if python is available in PATH - $pythonCmd = Get-Command "python${env:python_version_spec}" -ErrorAction SilentlyContinue - if ($pythonCmd) { - Write-Host "Found `python${env:python_version_spec}` at $($pythonCmd.Source)" - exit 0 - } - - $pythonCmd = Get-Command "python" -ErrorAction SilentlyContinue - if ($pythonCmd) { - Write-Host "Found `python` at $($pythonCmd.Source)." - # FUTURE WORK: validate it matches the requested version spec - exit 0 - } - - Write-Host "##vso[task.logissue type=error]Neither `python` nor `python${env:python_version_spec}` are on PATH." - exit 1 +- task: PowerShell@2 displayName: 'Verify Python ${{ parameters.versionSpec }} is on path' + inputs: + targetType: 'inline' + script: | + # Check if python is available in PATH + $pythonCmd = Get-Command "python${env:python_version_spec}" -ErrorAction SilentlyContinue + if ($pythonCmd) { + Write-Host "Found `python${env:python_version_spec}` at $($pythonCmd.Source)" + exit 0 + } + + $pythonCmd = Get-Command "python" -ErrorAction SilentlyContinue + if ($pythonCmd) { + Write-Host "Found `python` at $($pythonCmd.Source)." + # FUTURE WORK: validate it matches the requested version spec + exit 0 + } + + Write-Host "##vso[task.logissue type=error]Neither `python` nor `python${env:python_version_spec}` are on PATH." + exit 1 env: python_version_spec: ${{ parameters.versionSpec }} @@ -73,47 +76,63 @@ steps: # across all runs. # Proper fix would be to have a post-job task to clean up our junk, but we can't do that without a full task def. # (Or intrusively set GRADLE_USER_HOME env var to an agent temp dir. Maven doesn't have equiv facilities.) -- pwsh: | - # Force everything in Maven to go through & authenticate using the main feed. - # HACK: Does not escape `parameters.artifactFeeds`. Assumes pattern is [A-Za-z0-9]+. - # HACK: Assumes feed is a org-scoped feed in `aiinfra`. - $m2SettingsXmlPath = "~/.m2/settings.xml" - $m2SettingsContent = Get-Content $m2SettingsXmlPath -Raw - - if ($m2SettingsContent -match '') { - echo "##vso[task.logissue type=error]Configuration already contains ``, cannot inject feed mirror w/o corrupting existing config." - exit 1 - } - - echo "Injecting Maven mirrors section for 1ES feed" - $( $m2SettingsContent -replace '', "${env:central_maven_feed_name}1ES Feed Overridehttps://aiinfra.pkgs.visualstudio.com/_packaging/${env:central_maven_feed_name}/maven/v1*" ) | ` - Set-Content $m2SettingsXmlPath - - - # gradle - inject 1ES feed repo into all projects/buildscripts. Auth using creds from maven settings.xml. - # nb: see HACK note above. - $gradleInitDir = if ($env:GRADLE_HOME_DIR) { $env:GRADLE_HOME_DIR } else { "~/.gradle" } - if (!(Test-Path -Path "${gradleInitDir}/init.d")) { - New-Item -ItemType Directory -Path "${gradleInitDir}/init.d" - } - Set-Content -Path "${gradleInitDir}/init.d/1es-feed-inject.init.gradle" -Value @" - // Use official 1ES feed. Network isolation prevents us from reaching out directly to other feeds. - - def mavenSettingsServers = { - File mavenSettings = new File(System.getProperty("user.home"), ".m2/settings.xml") - return new XmlSlurper().parse(mavenSettings)."servers"."server" - } - - def oneEsFeedCredentials = { - for (x in mavenSettingsServers()) { - if (x."id".text() == "${env:central_maven_feed_name}") { - return [username: x.username.text(), password: x.password.text()] +- task: PowerShell@2 + displayName: 'Maven/Gradle - mirror / feed via `${{ parameters.artifactFeeds }}`' + inputs: + targetType: 'inline' + script: | + # Force everything in Maven to go through & authenticate using the main feed. + # HACK: Does not escape `parameters.artifactFeeds`. Assumes pattern is [A-Za-z0-9]+. + # HACK: Assumes feed is a org-scoped feed in `aiinfra`. + $m2SettingsXmlPath = "~/.m2/settings.xml" + $m2SettingsContent = Get-Content $m2SettingsXmlPath -Raw + + if ($m2SettingsContent -match '') { + echo "##vso[task.logissue type=error]Configuration already contains ``, cannot inject feed mirror w/o corrupting existing config." + exit 1 + } + + echo "Injecting Maven mirrors section for 1ES feed" + $( $m2SettingsContent -replace '', "${env:central_maven_feed_name}1ES Feed Overridehttps://aiinfra.pkgs.visualstudio.com/_packaging/${env:central_maven_feed_name}/maven/v1*" ) | ` + Set-Content $m2SettingsXmlPath + + + # gradle - inject 1ES feed repo into all projects/buildscripts. Auth using creds from maven settings.xml. + # nb: see HACK note above. + $gradleInitDir = if ($env:GRADLE_HOME_DIR) { $env:GRADLE_HOME_DIR } else { "~/.gradle" } + if (!(Test-Path -Path "${gradleInitDir}/init.d")) { + New-Item -ItemType Directory -Path "${gradleInitDir}/init.d" + } + Set-Content -Path "${gradleInitDir}/init.d/1es-feed-inject.init.gradle" -Value @" + // Use official 1ES feed. Network isolation prevents us from reaching out directly to other feeds. + + def mavenSettingsServers = { + File mavenSettings = new File(System.getProperty("user.home"), ".m2/settings.xml") + return new XmlSlurper().parse(mavenSettings)."servers"."server" + } + + def oneEsFeedCredentials = { + for (x in mavenSettingsServers()) { + if (x."id".text() == "${env:central_maven_feed_name}") { + return [username: x.username.text(), password: x.password.text()] + } } } - } - allprojects { - buildscript { + allprojects { + buildscript { + repositories { + maven { + name "1ES Feed" + url uri("https://aiinfra.pkgs.visualstudio.com/_packaging/${env:central_maven_feed_name}/maven/v1") + credentials(PasswordCredentials) { + username oneEsFeedCredentials().username + password oneEsFeedCredentials().password + } + } + } + } + repositories { maven { name "1ES Feed" @@ -125,42 +144,32 @@ steps: } } } + "@ + env: + central_maven_feed_name: ${{ parameters.artifactFeeds }} - repositories { - maven { - name "1ES Feed" - url uri("https://aiinfra.pkgs.visualstudio.com/_packaging/${env:central_maven_feed_name}/maven/v1") - credentials(PasswordCredentials) { - username oneEsFeedCredentials().username - password oneEsFeedCredentials().password - } - } - } - } - "@ - displayName: 'Maven/Gradle - mirror / feed via `${{ parameters.artifactFeeds }}`' - env: - central_maven_feed_name: ${{ parameters.artifactFeeds }} - -- pwsh: | - # NuGet appears to fail if any of the pkg sources are unreachable. - # Replace the NuGet.config w/ one that only has the internal feed. - # HACK: We'd rather splice the structure, not override the file. - Set-Content -Path "NuGet.config" -Value @" - - - - - - - - - - - - - - "@ +- task: PowerShell@2 displayName: NuGet - override `NuGet.config` to lock feed to `${{ parameters.artifactFeeds }}` + inputs: + targetType: 'inline' + script: | + # NuGet appears to fail if any of the pkg sources are unreachable. + # Replace the NuGet.config w/ one that only has the internal feed. + # HACK: We'd rather splice the structure, not override the file. + Set-Content -Path "NuGet.config" -Value @" + + + + + + + + + + + + + + "@ env: central_maven_feed_name: ${{ parameters.artifactFeeds }} From b16019c91edb8821a8c7688706e31562b4c2dba8 Mon Sep 17 00:00:00 2001 From: Erik Date: Wed, 15 Apr 2026 17:00:00 -0400 Subject: [PATCH 2/3] . --- .../stages/py-cpu-packaging-stage.yml | 65 ------------------- .../setup-feeds-and-python-steps.yml | 8 +-- 2 files changed, 4 insertions(+), 69 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml index 6beff17adc31c..25bded72891d6 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml @@ -107,68 +107,3 @@ stages: extra_build_arg: ${{ parameters.build_py_parameters }} cmake_build_type: ${{ parameters.cmake_build_type }} is1ES: true - -- ${{ if eq(parameters.enable_windows_arm64_qnn, true) }}: - - stage: Python_Packaging_Windows_ARM64_QNN - dependsOn: [] - jobs: - - template: ../templates/py-win-arm64-qnn.yml - parameters: - MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - PYTHON_VERSION: '3.11' - - - template: ../templates/py-win-arm64-qnn.yml - parameters: - MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - PYTHON_VERSION: '3.12' - - - template: ../templates/py-win-arm64-qnn.yml - parameters: - MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - PYTHON_VERSION: '3.13' - - - template: ../templates/py-win-arm64-qnn.yml - parameters: - MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - PYTHON_VERSION: '3.14' - -- ${{ if eq(parameters.enable_windows_arm64ec_qnn, true) }}: - - stage: Python_Packaging_Windows_arm64ec_QNN - dependsOn: [] - jobs: - - template: ../templates/py-win-arm64ec-qnn.yml - parameters: - MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - -- ${{ if eq(parameters.enable_windows_x64_qnn, true) }}: - - stage: Python_Packaging_Windows_x64_QNN - dependsOn: [] - jobs: - - template: ../templates/py-win-x64-qnn.yml - parameters: - MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - is1ES: true - -- ${{ if eq(parameters.enable_linux_x64_qnn, true) }}: - - stage: Python_Packaging_Linux_x64_QNN - dependsOn: [] - jobs: - - template: ../templates/py-linux-qnn.yml - parameters: - machine_pool: 'onnxruntime-Ubuntu2404-AMD-CPU' - QnnSdk: ${{ parameters.qnn_sdk_version }} - extra_build_arg: ${{ parameters.build_py_parameters }} - cmake_build_type: ${{ parameters.cmake_build_type }} - is1ES: true diff --git a/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml b/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml index ab424f397f0f5..3c5bf14440453 100644 --- a/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml @@ -78,6 +78,8 @@ steps: # (Or intrusively set GRADLE_USER_HOME env var to an agent temp dir. Maven doesn't have equiv facilities.) - task: PowerShell@2 displayName: 'Maven/Gradle - mirror / feed via `${{ parameters.artifactFeeds }}`' + env: + central_maven_feed_name: ${{ parameters.artifactFeeds }} inputs: targetType: 'inline' script: | @@ -145,11 +147,11 @@ steps: } } "@ - env: - central_maven_feed_name: ${{ parameters.artifactFeeds }} - task: PowerShell@2 displayName: NuGet - override `NuGet.config` to lock feed to `${{ parameters.artifactFeeds }}` + env: + central_maven_feed_name: ${{ parameters.artifactFeeds }} inputs: targetType: 'inline' script: | @@ -171,5 +173,3 @@ steps: "@ - env: - central_maven_feed_name: ${{ parameters.artifactFeeds }} From aa11b311b7715fd2a6939e0768b74fd3666534de Mon Sep 17 00:00:00 2001 From: Erik Date: Wed, 15 Apr 2026 17:18:08 -0400 Subject: [PATCH 3/3] Fix --- .../templates/setup-feeds-and-python-steps.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml b/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml index 3c5bf14440453..c2f170bed3c44 100644 --- a/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml @@ -23,8 +23,14 @@ steps: # HACK: Ensure there are no leftover m2 settings from previous runs. Agent pools are currently stateful. # Ideally we'd just push/pop/layer specific splices to the settings (auth & mirror stuff), but there's no easy # mechanism for that AFAIK. -- pwsh: if (Test-Path ~/.m2/settings.xml) { Remove-Item ~/.m2/settings.xml } +- task: PowerShell@2 displayName: Maven - remove existing ~/.m2/settings.xml + inputs: + targetType: 'inline' + script: | + if (Test-Path ~/.m2/settings.xml) { + Remove-Item ~/.m2/settings.xml + } - task: NuGetAuthenticate@1 # just auth w/ internal feeds