Add a Memory Profiling build job in post merge pipeline (#16172)

### Description 1. Add a Memory Profiling build job 2. Remove no absl build job since the feature will be removed 3. Simplify post-merge-jobs.yml by unifying the pool names ### Motivation and Context To catch build errors in #16124
microsoft · Jun 1, 2023 · 5bfa118 · 5bfa118
1 parent e6c6184
commit 5bfa118
Show file tree

Hide file tree

Showing 14 changed files with 28 additions and 39 deletions.
diff --git a/tools/ci_build/github/azure-pipelines/build-perf-test-binaries-pipeline.yml b/tools/ci_build/github/azure-pipelines/build-perf-test-binaries-pipeline.yml
@@ -28,7 +28,7 @@ stages:
         artifactName: 'onnxruntime-android-full-aar'
         job_name_suffix: 'Full'
         publish_executables: '1'
-        pool_name: 'aiinfra-Linux-CPU'
+        pool_name: 'onnxruntime-Ubuntu2004-AMD-CPU'
 
 # build Python packages
 # Linux GPU only

diff --git a/tools/ci_build/github/azure-pipelines/nodejs/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nodejs/templates/test_linux.yml
@@ -1,5 +1,5 @@
 parameters:
-  AgentPool: 'aiinfra-Linux-CPU'
+  AgentPool: 'onnxruntime-Ubuntu2004-AMD-CPU'
   StageSuffix: ''
 stages:
 - stage: Nodejs_Test_${{ parameters.StageSuffix }}

diff --git a/tools/ci_build/github/azure-pipelines/npm-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/npm-packaging-pipeline.yml
@@ -49,7 +49,7 @@ stages:
   parameters:
     NpmPackagingMode: ${{ variables.NpmPackagingMode }}
     BuildConfig: 'Release'
-    PoolName: 'aiinfra-Linux-CPU'
+    PoolName: 'onnxruntime-Ubuntu2004-AMD-CPU'
     PackageName: 'onnxruntime-react-native'
     BuildAndroidAARStageDependsOn: 'Extract_commit'
 

diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml
@@ -1,5 +1,5 @@
 parameters:
-  AgentPool: 'aiinfra-Linux-CPU'
+  AgentPool: 'onnxruntime-Ubuntu2004-AMD-CPU'
   ArtifactSuffix: ''
   NugetPackageName : ''
   StageSuffix: 'CPU'

diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml
@@ -25,7 +25,7 @@ stages:
       timeoutInMinutes: 120
       workspace:
         clean: all
-      pool: aiinfra-Linux-CPU
+      pool: onnxruntime-Ubuntu2004-AMD-CPU
 
       strategy:
         matrix:

diff --git a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml
@@ -1,14 +1,3 @@
-variables:
-- name: WIN_CPU_BUILD_MACHINE_POOL_NAME
-  value: 'onnxruntime-Win-CPU-2022'
-- name: LINUX_CPU_BUILD_MACHINE_POOL_NAME  # name of a variable
-  # The public ADO project
-  ${{ if startsWith(variables['System.CollectionUri'], 'https://dev.azure.com/onnxruntime/') }}:
-    value: 'onnxruntime-Ubuntu2004-AMD-CPU'
-  # The private ADO project
-  ${{ if or(startsWith(variables['System.CollectionUri'], 'https://dev.azure.com/aiinfra/'),startsWith(variables['System.CollectionUri'], 'https://aiinfra.visualstudio.com/')) }}:
-    value: 'aiinfra-Linux-CPU'
-
 stages:
 - ${{ if or(startsWith(variables['System.CollectionUri'], 'https://dev.azure.com/aiinfra/'),startsWith(variables['System.CollectionUri'], 'https://aiinfra.visualstudio.com/')) }}:
   - template: templates/web-ci.yml
@@ -38,7 +27,7 @@ stages:
     runTests: true
     buildJava: false
     buildNodejs: false
-    ort_build_pool_name: ${{variables.WIN_CPU_BUILD_MACHINE_POOL_NAME}}
+    ort_build_pool_name: 'onnxruntime-Win-CPU-2022'
 
 - template: templates/win-ci.yml
   parameters:
@@ -53,7 +42,7 @@ stages:
     runTests: false
     buildJava: false
     buildNodejs: true
-    ort_build_pool_name: ${{variables.WIN_CPU_BUILD_MACHINE_POOL_NAME}}
+    ort_build_pool_name: 'onnxruntime-Win-CPU-2022'
 
 - template: templates/win-ci.yml
   parameters:
@@ -68,7 +57,7 @@ stages:
     runTests: true
     buildJava: true
     buildNodejs: true
-    ort_build_pool_name: ${{variables.WIN_CPU_BUILD_MACHINE_POOL_NAME}}
+    ort_build_pool_name: 'onnxruntime-Win-CPU-2022'
 
 - stage: Mimalloc
   dependsOn: [ ]
@@ -88,27 +77,27 @@ stages:
       ORT_EP_NAME: CPU
       GenerateDocumentation: false
       EnablePython: false
-      MachinePool: ${{variables.WIN_CPU_BUILD_MACHINE_POOL_NAME}}
+      MachinePool: 'onnxruntime-Win-CPU-2022'
 
-- stage: NoAbseil
+- stage: MemoryProfiling
   dependsOn: [ ]
   jobs:
   - template: templates/jobs/win-ci-vs-2022-job.yml
     parameters:
       BuildConfig: 'Debug'
       EnvSetupScript: setup_env.bat
       buildArch: x64
-      additionalBuildFlags: --cmake_extra_defines onnxruntime_DISABLE_ABSEIL=ON
+      additionalBuildFlags: --cmake_extra_defines onnxruntime_ENABLE_MEMORY_PROFILE=ON
       msbuildPlatform: x64
       isX86: false
-      job_name_suffix: x64_no_absl
-      RunOnnxRuntimeTests: true
+      job_name_suffix: x64_no_memory_profiling
+      RunOnnxRuntimeTests: false
       RunStaticCodeAnalysis: false
       isTraining: false
       ORT_EP_NAME: CPU
       GenerateDocumentation: false
       EnablePython: false
-      MachinePool: ${{variables.WIN_CPU_BUILD_MACHINE_POOL_NAME}}
+      MachinePool: 'onnxruntime-Win-CPU-2022'
 
 - stage: MinimalBuildWithNoExceptions
   dependsOn: [ ]
@@ -128,7 +117,7 @@ stages:
       ORT_EP_NAME: CPU
       GenerateDocumentation: false
       EnablePython: false
-      MachinePool: ${{variables.WIN_CPU_BUILD_MACHINE_POOL_NAME}}
+      MachinePool: 'onnxruntime-Win-CPU-2022'
 
 - stage: DebugNodeInputsOutputs
   dependsOn: [ ]
@@ -148,7 +137,7 @@ stages:
       ORT_EP_NAME: CPU
       GenerateDocumentation: false
       EnablePython: false
-      MachinePool: ${{variables.WIN_CPU_BUILD_MACHINE_POOL_NAME}}
+      MachinePool: 'onnxruntime-Win-CPU-2022'
 
 #Generate test coverage report and publish the data to a Cloud database. Only runs daily.
 - stage: CodeCoverage
@@ -160,7 +149,7 @@ stages:
     timeoutInMinutes: 150
     variables:
       skipComponentGovernanceDetection: true
-    pool: ${{variables.LINUX_CPU_BUILD_MACHINE_POOL_NAME}}
+    pool: 'onnxruntime-Ubuntu2004-AMD-CPU'
     steps:
     - template: templates/set-version-number-variables-step.yml
 
@@ -206,7 +195,7 @@ stages:
   - job: AndroidCustomBuildScript
     workspace:
       clean: all
-    pool: ${{variables.LINUX_CPU_BUILD_MACHINE_POOL_NAME}}
+    pool: 'onnxruntime-Ubuntu2004-AMD-CPU'
     variables:
       dockerImageTag: onnxruntime-android-custom-build
     steps:

diff --git a/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml
@@ -10,7 +10,7 @@ stages:
   - template: templates/py-packaging-linux-test.yml
     parameters:
       arch: 'x86_64'
-      machine_pool: 'aiinfra-Linux-CPU'
+      machine_pool: 'onnxruntime-Ubuntu2004-AMD-CPU'
       device: 'CPU'
 
 - stage: Linux_Test_CPU_aarch64_stage

diff --git a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml
@@ -33,7 +33,7 @@ parameters:
 - name: pool_name
   displayName: Pool name
   type: string
-  default: 'aiinfra-Linux-CPU'
+  default: 'onnxruntime-Ubuntu2004-AMD-CPU'
 
 - name: packageName
   # now we can build onnxruntime or onnxruntime-mobile for Android, need specify it here

diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml
@@ -780,7 +780,7 @@ stages:
 
 - template: ../nuget/templates/test_linux.yml
   parameters:
-    AgentPool : aiinfra-Linux-CPU
+    AgentPool : onnxruntime-Ubuntu2004-AMD-CPU
     NugetPackageName : 'Microsoft.ML.OnnxRuntime'
     ArtifactSuffix: 'CPU'
     SpecificArtifact: ${{ parameters.SpecificArtifact }}
@@ -798,7 +798,7 @@ stages:
 
 - template: ../nodejs/templates/test_linux.yml
   parameters:
-    AgentPool : 'aiinfra-Linux-CPU'
+    AgentPool : 'onnxruntime-Ubuntu2004-AMD-CPU'
     StageSuffix : 'Linux_CPU_x64'
 
 - template: ../nodejs/templates/test_macos.yml
@@ -852,7 +852,7 @@ stages:
   - job:
     workspace:
       clean: all
-    pool: 'aiinfra-Linux-CPU'
+    pool: 'onnxruntime-Ubuntu2004-AMD-CPU'
     variables:
     - name: runCodesignValidationInjection
       value: false

diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml
@@ -22,7 +22,7 @@ parameters:
 
 - name: PoolName
   type: string
-  default: 'aiinfra-Linux-CPU'
+  default: 'onnxruntime-Ubuntu2004-AMD-CPU'
 
 - name: ArtifactNamePrefix
   type: string

diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-cpu-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/templates/linux-cpu-packaging-pipeline.yml
@@ -34,7 +34,7 @@ stages:
       OnnxruntimeCFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
       OnnxruntimeCXXFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
       OnnxruntimeNodejsBindingArch: 'x64'
-      PoolName: 'aiinfra-Linux-CPU'
+      PoolName: 'onnxruntime-Ubuntu2004-AMD-CPU'
       ArtifactNamePrefix: ${{ parameters.ArtifactNamePrefix }}
       PackageJava: ${{ parameters.PackageJava }}
       PackageNodeJS: ${{ parameters.PackageNodeJS }}

diff --git a/tools/ci_build/github/azure-pipelines/templates/ondevice-training-cpu-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/templates/ondevice-training-cpu-packaging-pipeline.yml
@@ -364,7 +364,7 @@ stages:
 
 - template: ../nuget/templates/test_linux.yml
   parameters:
-    AgentPool : aiinfra-Linux-CPU
+    AgentPool : onnxruntime-Ubuntu2004-AMD-CPU
     NugetPackageName : 'Microsoft.ML.OnnxRuntime.Training'
     ArtifactSuffix: 'Training-CPU'
     StageSuffix: 'Training_CPU'

diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-selectable-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-selectable-stage.yml
@@ -48,7 +48,7 @@ stages:
       timeoutInMinutes: 90
       workspace:
         clean: all
-      pool: aiinfra-Linux-CPU
+      pool: onnxruntime-Ubuntu2004-AMD-CPU
       strategy:
         matrix:
           ${{ each PythonVersion in parameters.python_version }}:

diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
@@ -513,6 +513,6 @@ stages:
       - template: py-linux-gpu.yml
         parameters:
           arch: 'x86_64'
-          machine_pool: 'aiinfra-Linux-CPU'
+          machine_pool: 'onnxruntime-Ubuntu2004-AMD-CPU'
           ${{ if contains(parameters.build_py_parameters, '--use_azure') }}:
             device: '-d AZURE'