Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 0 additions & 30 deletions tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,26 +19,6 @@ parameters:
type: boolean
default: true

- name: enable_windows_arm64_qnn
displayName: 'Whether Windows ARM64 package with QNN EP is built.'
type: boolean
default: true

- name: enable_windows_arm64ec_qnn
displayName: 'Whether Windows ARM64EC package with QNN EP is built.'
type: boolean
default: true

- name: enable_windows_x64_qnn
displayName: 'Whether Windows x86_64 package with QNN EP is built.'
type: boolean
default: true

- name: enable_linux_x64_qnn
displayName: 'Whether Linux x86_64 package with QNN EP is built.'
type: boolean
default: true

- name: build_py_parameters
displayName: 'Specify extra build parameters'
type: string
Expand All @@ -55,11 +35,6 @@ parameters:
- RelWithDebInfo
- MinSizeRel

# Only applies to QNN packages.
- name: qnn_sdk_version
type: string
displayName: 'QNN SDK version. Only for QNN packages.'
default: 2.42.0.251225

trigger: none

Expand Down Expand Up @@ -103,10 +78,5 @@ extends:
enable_windows_cpu: ${{ parameters.enable_windows_cpu }}
enable_mac_cpu: ${{ parameters.enable_mac_cpu }}
enable_linux_arm: ${{ parameters.enable_linux_arm }}
enable_windows_arm64_qnn: ${{ parameters.enable_windows_arm64_qnn }}
enable_windows_arm64ec_qnn: ${{ parameters.enable_windows_arm64ec_qnn }}
enable_windows_x64_qnn: ${{ parameters.enable_windows_x64_qnn }}
enable_linux_x64_qnn: ${{ parameters.enable_linux_x64_qnn }}
build_py_parameters: ${{ parameters.build_py_parameters }}
cmake_build_type: ${{ parameters.cmake_build_type }}
qnn_sdk_version: ${{ parameters.qnn_sdk_version }}
Original file line number Diff line number Diff line change
Expand Up @@ -25,26 +25,6 @@ parameters:
type: boolean
default: true

- name: enable_windows_arm64_qnn
displayName: 'Whether Windows ARM64 package with QNN EP is built.'
type: boolean
default: true

- name: enable_windows_arm64ec_qnn
displayName: 'Whether Windows ARM64EC package with QNN EP is built.'
type: boolean
default: true

- name: enable_windows_x64_qnn
displayName: 'Whether Windows x86_64 package with QNN EP is built.'
type: boolean
default: true

- name: enable_linux_x64_qnn
displayName: 'Whether Linux x86_64 package with QNN EP is built.'
type: boolean
default: true

- name: cmake_build_type
type: string
displayName: 'Linux packages cmake build type. Linux Only.'
Expand All @@ -55,12 +35,6 @@ parameters:
- RelWithDebInfo
- MinSizeRel

# Only applies to QNN packages.
- name: qnn_sdk_version
type: string
displayName: 'QNN SDK version. Only for QNN packages.'
default: 2.42.0.251225

stages:
- ${{ if eq(parameters.enable_windows_cpu, true) }}:
- stage: Python_Packaging_Windows_CPU
Expand Down Expand Up @@ -133,68 +107,3 @@ stages:
extra_build_arg: ${{ parameters.build_py_parameters }}
cmake_build_type: ${{ parameters.cmake_build_type }}
is1ES: true

- ${{ if eq(parameters.enable_windows_arm64_qnn, true) }}:
- stage: Python_Packaging_Windows_ARM64_QNN
dependsOn: []
jobs:
- template: ../templates/py-win-arm64-qnn.yml
parameters:
MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64'
QNN_SDK: ${{ parameters.qnn_sdk_version }}
BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }}
PYTHON_VERSION: '3.11'

- template: ../templates/py-win-arm64-qnn.yml
parameters:
MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64'
QNN_SDK: ${{ parameters.qnn_sdk_version }}
BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }}
PYTHON_VERSION: '3.12'

- template: ../templates/py-win-arm64-qnn.yml
parameters:
MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64'
QNN_SDK: ${{ parameters.qnn_sdk_version }}
BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }}
PYTHON_VERSION: '3.13'

- template: ../templates/py-win-arm64-qnn.yml
parameters:
MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64'
QNN_SDK: ${{ parameters.qnn_sdk_version }}
BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }}
PYTHON_VERSION: '3.14'

- ${{ if eq(parameters.enable_windows_arm64ec_qnn, true) }}:
- stage: Python_Packaging_Windows_arm64ec_QNN
dependsOn: []
jobs:
- template: ../templates/py-win-arm64ec-qnn.yml
parameters:
MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU'
QNN_SDK: ${{ parameters.qnn_sdk_version }}
BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }}

- ${{ if eq(parameters.enable_windows_x64_qnn, true) }}:
- stage: Python_Packaging_Windows_x64_QNN
dependsOn: []
jobs:
- template: ../templates/py-win-x64-qnn.yml
parameters:
MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU'
QNN_SDK: ${{ parameters.qnn_sdk_version }}
BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }}
is1ES: true

- ${{ if eq(parameters.enable_linux_x64_qnn, true) }}:
- stage: Python_Packaging_Linux_x64_QNN
dependsOn: []
jobs:
- template: ../templates/py-linux-qnn.yml
parameters:
machine_pool: 'onnxruntime-Ubuntu2404-AMD-CPU'
QnnSdk: ${{ parameters.qnn_sdk_version }}
extra_build_arg: ${{ parameters.build_py_parameters }}
cmake_build_type: ${{ parameters.cmake_build_type }}
is1ES: true
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,14 @@ steps:
# HACK: Ensure there are no leftover m2 settings from previous runs. Agent pools are currently stateful.
# Ideally we'd just push/pop/layer specific splices to the settings (auth & mirror stuff), but there's no easy
# mechanism for that AFAIK.
- pwsh: if (Test-Path ~/.m2/settings.xml) { Remove-Item ~/.m2/settings.xml }
- task: PowerShell@2
displayName: Maven - remove existing ~/.m2/settings.xml
inputs:
targetType: 'inline'
script: |
if (Test-Path ~/.m2/settings.xml) {
Remove-Item ~/.m2/settings.xml
}

- task: NuGetAuthenticate@1 # just auth w/ internal feeds

Expand All @@ -48,72 +54,93 @@ steps:
${{ if ne(parameters.architecture, '') }}:
architecture: ${{ parameters.architecture }}

- pwsh: |
# Check if python is available in PATH
$pythonCmd = Get-Command "python${env:python_version_spec}" -ErrorAction SilentlyContinue
if ($pythonCmd) {
Write-Host "Found `python${env:python_version_spec}` at $($pythonCmd.Source)"
exit 0
}

$pythonCmd = Get-Command "python" -ErrorAction SilentlyContinue
if ($pythonCmd) {
Write-Host "Found `python` at $($pythonCmd.Source)."
# FUTURE WORK: validate it matches the requested version spec
exit 0
}

Write-Host "##vso[task.logissue type=error]Neither `python` nor `python${env:python_version_spec}` are on PATH."
exit 1
- task: PowerShell@2
displayName: 'Verify Python ${{ parameters.versionSpec }} is on path'
inputs:
targetType: 'inline'
script: |
# Check if python is available in PATH
$pythonCmd = Get-Command "python${env:python_version_spec}" -ErrorAction SilentlyContinue
if ($pythonCmd) {
Write-Host "Found `python${env:python_version_spec}` at $($pythonCmd.Source)"
exit 0
}

$pythonCmd = Get-Command "python" -ErrorAction SilentlyContinue
if ($pythonCmd) {
Write-Host "Found `python` at $($pythonCmd.Source)."
# FUTURE WORK: validate it matches the requested version spec
exit 0
}

Write-Host "##vso[task.logissue type=error]Neither `python` nor `python${env:python_version_spec}` are on PATH."
exit 1
env:
python_version_spec: ${{ parameters.versionSpec }}

# HACK: If agent pools are stateful then we have to ensure that `1es-feed-inject.init.gradle` file path is constant
# across all runs.
# Proper fix would be to have a post-job task to clean up our junk, but we can't do that without a full task def.
# (Or intrusively set GRADLE_USER_HOME env var to an agent temp dir. Maven doesn't have equiv facilities.)
- pwsh: |
# Force everything in Maven to go through & authenticate using the main feed.
# HACK: Does not escape `parameters.artifactFeeds`. Assumes pattern is [A-Za-z0-9]+.
# HACK: Assumes feed is a org-scoped feed in `aiinfra`.
$m2SettingsXmlPath = "~/.m2/settings.xml"
$m2SettingsContent = Get-Content $m2SettingsXmlPath -Raw

if ($m2SettingsContent -match '<mirrors>') {
echo "##vso[task.logissue type=error]Configuration already contains `<mirrors>`, cannot inject feed mirror w/o corrupting existing config."
exit 1
}

echo "Injecting Maven mirrors section for 1ES feed"
$( $m2SettingsContent -replace '</settings>', "<mirrors><mirror><id>${env:central_maven_feed_name}</id><name>1ES Feed Override</name><url>https://aiinfra.pkgs.visualstudio.com/_packaging/${env:central_maven_feed_name}/maven/v1</url><mirrorOf>*</mirrorOf></mirror></mirrors></settings>" ) | `
Set-Content $m2SettingsXmlPath


# gradle - inject 1ES feed repo into all projects/buildscripts. Auth using creds from maven settings.xml.
# nb: see HACK note above.
$gradleInitDir = if ($env:GRADLE_HOME_DIR) { $env:GRADLE_HOME_DIR } else { "~/.gradle" }
if (!(Test-Path -Path "${gradleInitDir}/init.d")) {
New-Item -ItemType Directory -Path "${gradleInitDir}/init.d"
}
Set-Content -Path "${gradleInitDir}/init.d/1es-feed-inject.init.gradle" -Value @"
// Use official 1ES feed. Network isolation prevents us from reaching out directly to other feeds.

def mavenSettingsServers = {
File mavenSettings = new File(System.getProperty("user.home"), ".m2/settings.xml")
return new XmlSlurper().parse(mavenSettings)."servers"."server"
}

def oneEsFeedCredentials = {
for (x in mavenSettingsServers()) {
if (x."id".text() == "${env:central_maven_feed_name}") {
return [username: x.username.text(), password: x.password.text()]
- task: PowerShell@2
displayName: 'Maven/Gradle - mirror / feed via `${{ parameters.artifactFeeds }}`'
env:
central_maven_feed_name: ${{ parameters.artifactFeeds }}
inputs:
targetType: 'inline'
script: |
# Force everything in Maven to go through & authenticate using the main feed.
# HACK: Does not escape `parameters.artifactFeeds`. Assumes pattern is [A-Za-z0-9]+.
# HACK: Assumes feed is a org-scoped feed in `aiinfra`.
$m2SettingsXmlPath = "~/.m2/settings.xml"
$m2SettingsContent = Get-Content $m2SettingsXmlPath -Raw

if ($m2SettingsContent -match '<mirrors>') {
echo "##vso[task.logissue type=error]Configuration already contains `<mirrors>`, cannot inject feed mirror w/o corrupting existing config."
exit 1
}

echo "Injecting Maven mirrors section for 1ES feed"
$( $m2SettingsContent -replace '</settings>', "<mirrors><mirror><id>${env:central_maven_feed_name}</id><name>1ES Feed Override</name><url>https://aiinfra.pkgs.visualstudio.com/_packaging/${env:central_maven_feed_name}/maven/v1</url><mirrorOf>*</mirrorOf></mirror></mirrors></settings>" ) | `
Set-Content $m2SettingsXmlPath


# gradle - inject 1ES feed repo into all projects/buildscripts. Auth using creds from maven settings.xml.
# nb: see HACK note above.
$gradleInitDir = if ($env:GRADLE_HOME_DIR) { $env:GRADLE_HOME_DIR } else { "~/.gradle" }
if (!(Test-Path -Path "${gradleInitDir}/init.d")) {
New-Item -ItemType Directory -Path "${gradleInitDir}/init.d"
}
Set-Content -Path "${gradleInitDir}/init.d/1es-feed-inject.init.gradle" -Value @"
// Use official 1ES feed. Network isolation prevents us from reaching out directly to other feeds.

def mavenSettingsServers = {
File mavenSettings = new File(System.getProperty("user.home"), ".m2/settings.xml")
return new XmlSlurper().parse(mavenSettings)."servers"."server"
}

def oneEsFeedCredentials = {
for (x in mavenSettingsServers()) {
if (x."id".text() == "${env:central_maven_feed_name}") {
return [username: x.username.text(), password: x.password.text()]
}
}
}
}

allprojects {
buildscript {
allprojects {
buildscript {
repositories {
maven {
name "1ES Feed"
url uri("https://aiinfra.pkgs.visualstudio.com/_packaging/${env:central_maven_feed_name}/maven/v1")
credentials(PasswordCredentials) {
username oneEsFeedCredentials().username
password oneEsFeedCredentials().password
}
}
}
}

repositories {
maven {
name "1ES Feed"
Expand All @@ -125,42 +152,30 @@ steps:
}
}
}
"@

repositories {
maven {
name "1ES Feed"
url uri("https://aiinfra.pkgs.visualstudio.com/_packaging/${env:central_maven_feed_name}/maven/v1")
credentials(PasswordCredentials) {
username oneEsFeedCredentials().username
password oneEsFeedCredentials().password
}
}
}
}
"@
displayName: 'Maven/Gradle - mirror / feed via `${{ parameters.artifactFeeds }}`'
env:
central_maven_feed_name: ${{ parameters.artifactFeeds }}

- pwsh: |
# NuGet appears to fail if any of the pkg sources are unreachable.
# Replace the NuGet.config w/ one that only has the internal feed.
# HACK: We'd rather splice the structure, not override the file.
Set-Content -Path "NuGet.config" -Value @"
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<solution>
<add key="disableSourceControlIntegration" value="true" />
</solution>
<packageSources>
<clear />
<add key="1ES Feed" value="https://pkgs.dev.azure.com/aiinfra/_packaging/${env:central_maven_feed_name}/nuget/v3/index.json" />
</packageSources>
<disabledPackageSources>
<clear />
</disabledPackageSources>
</configuration>
"@
- task: PowerShell@2
displayName: NuGet - override `NuGet.config` to lock feed to `${{ parameters.artifactFeeds }}`
env:
central_maven_feed_name: ${{ parameters.artifactFeeds }}
inputs:
targetType: 'inline'
script: |
# NuGet appears to fail if any of the pkg sources are unreachable.
# Replace the NuGet.config w/ one that only has the internal feed.
# HACK: We'd rather splice the structure, not override the file.
Set-Content -Path "NuGet.config" -Value @"
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<solution>
<add key="disableSourceControlIntegration" value="true" />
</solution>
<packageSources>
<clear />
<add key="1ES Feed" value="https://pkgs.dev.azure.com/aiinfra/_packaging/${env:central_maven_feed_name}/nuget/v3/index.json" />
</packageSources>
<disabledPackageSources>
<clear />
</disabledPackageSources>
</configuration>
"@
Loading