Make Cuda packaging pipeline 1ES compliant

tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml

@@ -95,7 +95,6 @@ stages:
 - template: templates/c-api-cpu.yml
   parameters:
     RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
-    DoCompliance: ${{ parameters.DoCompliance }}
     DoEsrp: ${{ parameters.DoEsrp }}
     IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
     ${{ if eq(parameters.NugetPackageSuffix, 'NONE') }}:
@@ -117,7 +116,6 @@ stages:
 
 - template: stages/nuget-combine-cuda-stage.yml
   parameters:
-      DoCompliance: ${{ parameters.DoCompliance }}
       CudaVersion: 11.8
       RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
       UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}

tools/ci_build/github/azure-pipelines/cuda-packaging-pipeline.yml

@@ -1,125 +1,133 @@
 parameters:
-  - name: RunOnnxRuntimeTests
-    displayName: Run Tests?
-    type: boolean
-    default: true
-
-  - name: UseIncreasedTimeoutForTests
-    displayName: Increase timeout for tests? Set it to false if you are doing an Onnx Runtime release.
-    type: boolean
-    default: false
-
-  - name: DoCompliance
-    displayName: Run Compliance Tasks?
-    type: boolean
-    default: true
-
-  - name: DoEsrp
-    displayName: Run code sign tasks? Must be true if you are doing an ONNX Runtime release
-    type: boolean
-    default: true
-
-  - name: IsReleaseBuild
-    displayName: Is a release build? Set it to true if you are doing an ONNX Runtime release.
-    type: boolean
-    default: false
-
-  - name: PreReleaseVersionSuffixString
-    displayName: Suffix added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the type of pre-release package.
-    type: string
-    values:
-      - alpha
-      - beta
-      - rc
-      - none
-    default: none
-
-  - name: PreReleaseVersionSuffixNumber
-    displayName: Number added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the sequence of a pre-release package.
-    type: number
-    default: 0
-
-  # these 2 parameters are used for debugging.
-  - name: SpecificArtifact
-    displayName: Use Specific Artifact (Debugging only)
-    type: boolean
-    default: false
-
-  - name: BuildId
-    displayName: Pipeline BuildId, you could find it in the URL
-    type: string
-    default: '0'
-
-  - name: CudaVersion
-    displayName: CUDA version
-    type: string
-    default: '12.2'
-    values:
-      - 11.8
-      - 12.2
+- name: RunOnnxRuntimeTests
+  displayName: Run Tests?
+  type: boolean
+  default: true
+
+- name: UseIncreasedTimeoutForTests
+  displayName: Increase timeout for tests? Set it to false if you are doing an Onnx Runtime release.
+  type: boolean
+  default: false
+
+- name: DoEsrp
+  displayName: Run code sign tasks? Must be true if you are doing an ONNX Runtime release
+  type: boolean
+  default: true
+
+- name: IsReleaseBuild
+  displayName: Is a release build? Set it to true if you are doing an ONNX Runtime release.
+  type: boolean
+  default: false
+
+- name: PreReleaseVersionSuffixString
+  displayName: Suffix added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the type of pre-release package.
+  type: string
+  values:
+  - alpha
+  - beta
+  - rc
+  - none
+  default: none
+
+- name: PreReleaseVersionSuffixNumber
+  displayName: Number added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the sequence of a pre-release package.
+  type: number
+  default: 0
+
+# these 2 parameters are used for debugging.
+- name: SpecificArtifact
+  displayName: Use Specific Artifact (Debugging only)
+  type: boolean
+  default: false
+
+- name: BuildId
+  displayName: Pipeline BuildId, you could find it in the URL
+  type: string
+  default: '0'
+
+- name: CudaVersion
+  displayName: CUDA version
+  type: string
+  default: '12.2'
+  values:
+  - 11.8
+  - 12.2
 
 variables:
-  - template: templates/common-variables.yml
-  - name: ReleaseVersionSuffix
-    value: ''
-  - name: win_trt_home
-    ${{ if eq(parameters.CudaVersion, '11.8') }}:
-      value: $(Agent.TempDirectory)\${{ variables.win_trt_folder_cuda11 }}
-    ${{ if eq(parameters.CudaVersion, '12.2') }}:
-      value: $(Agent.TempDirectory)\${{ variables.win_trt_folder_cuda12 }}
-  - name: win_cuda_home
-    ${{ if eq(parameters.CudaVersion, '11.8') }}:
-      value: $(Agent.TempDirectory)\v11.8
-    ${{ if eq(parameters.CudaVersion, '12.2') }}:
-      value: $(Agent.TempDirectory)\v12.2
+- template: templates/common-variables.yml
+- name: ReleaseVersionSuffix
+  value: ''
+- name: win_trt_home
+  ${{ if eq(parameters.CudaVersion, '11.8') }}:
+    value: $(Agent.TempDirectory)\${{ variables.win_trt_folder_cuda11 }}
+  ${{ if eq(parameters.CudaVersion, '12.2') }}:
+    value: $(Agent.TempDirectory)\${{ variables.win_trt_folder_cuda12 }}
+- name: win_cuda_home
+  ${{ if eq(parameters.CudaVersion, '11.8') }}:
+    value: $(Agent.TempDirectory)\v11.8
+  ${{ if eq(parameters.CudaVersion, '12.2') }}:
+    value: $(Agent.TempDirectory)\v12.2
+
 resources:
   repositories:
-    - repository: onnxruntime-inference-examples # The name used to reference this repository in the checkout step
-      type: github
-      endpoint: ort-examples
-      name: microsoft/onnxruntime-inference-examples
-
-
-stages:
-  # Set ReleaseVersionSuffix
-  - template: stages/set_packaging_variables_stage.yml
-    parameters:
-      IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
-      PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }}
-      PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }}
-
-  # this is needed for certain artifacts to be published
-  - stage: Linux_C_API_Packaging_CPU
-    dependsOn: [ ]
-    jobs:
-    - template: templates/c-api-linux-cpu.yml
+  - repository: onnxruntime-inference-examples # The name used to reference this repository in the checkout step
+    type: github
+    endpoint: ort-examples
+    name: microsoft/onnxruntime-inference-examples
+  - repository: 1esPipelines
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+
+extends:
+  # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
+  # For non-production pipelines, use "Unofficial" as defined below.
+  # For productions pipelines, use "Official".
+  template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
+  parameters:
+    sdl:
+      binskim:
+        enabled: true
+        analyzeTargetGlob: $(Build.ArtifactStagingDirectory)/**.dll
+      sourceAnalysisPool:
+        name: onnxruntime-Win-CPU-2022
+        os: windows
+      sourceRepositoriesToScan:
+        exclude:
+        - repository: onnxruntime-inference-examples
+      spotBugs:
+        enabled: false
+        justificationForDisabling: "Getting ##[error1. SpotBugs Error gdn.unknownFormatResult - File: spotbugs.xml, which indicates that SpotBugs found one or more errors, which are not handled by the Guardian right now."
+    stages:
+    # Set ReleaseVersionSuffix
+    - template: stages/set_packaging_variables_stage.yml
       parameters:
-        OnnxruntimeArch: 'x64'
-        OnnxruntimeNodejsBindingArch: 'x64'
-        PoolName: 'onnxruntime-Ubuntu2204-AMD-CPU'
-        PackageJava: false
-        PackageNodeJS: false
-
-  # Nuget Packaging
-  - template: stages/nuget-combine-cuda-stage.yml
-    parameters:
-      DoCompliance: ${{ parameters.DoCompliance }}
-      CudaVersion: ${{ parameters.CudaVersion }}
-      RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
-      UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
-      win_trt_home: ${{ variables.win_trt_home }}
-      win_cuda_home: ${{ variables.win_cuda_home }}
-      DoEsrp: ${{ parameters.DoEsrp }}
-      IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
-      buildJava: true
-      buildNodejs: false
-      SpecificArtifact: ${{ parameters.SpecificArtifact }}
-      BuildId: ${{ parameters.BuildId }}
+        IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
+        PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }}
+        PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }}
 
-  - template: stages/download-java-tools-stage.yml
+    # this is needed for certain artifacts to be published
+    - template: stages/c-api-linux-cpu-stage.yml
 
-  - template: stages/java-cuda-packaging-stage.yml
-    parameters:
-      CudaVersion: ${{ parameters.CudaVersion }}
-      SpecificArtifact: ${{ parameters.SpecificArtifact }}
-      BuildId: ${{ parameters.BuildId }}
+    # Nuget Packaging
+    - template: stages/nuget-combine-cuda-stage.yml
+      parameters:
+        CudaVersion: ${{ parameters.CudaVersion }}
+        RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
+        UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
+        win_trt_home: ${{ variables.win_trt_home }}
+        win_cuda_home: ${{ variables.win_cuda_home }}
+        DoEsrp: ${{ parameters.DoEsrp }}
+        IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
+        buildJava: true
+        buildNodejs: false
+        SpecificArtifact: ${{ parameters.SpecificArtifact }}
+        BuildId: ${{ parameters.BuildId }}
+
+    - template: stages/download-java-tools-stage.yml
+
+    - template: stages/java-cuda-packaging-stage.yml
+      parameters:
+        CudaVersion: ${{ parameters.CudaVersion }}
+        SpecificArtifact: ${{ parameters.SpecificArtifact }}
+        BuildId: ${{ parameters.BuildId }}

tools/ci_build/github/azure-pipelines/nodejs/templates/test_linux.yml

@@ -10,8 +10,10 @@ stages:
   - job: Nodejs_Test_${{ parameters.StageSuffix }}
     workspace:
       clean: all
-    timeoutInMinutes:  120
-    pool: ${{ parameters.AgentPool }}
+    timeoutInMinutes: 120
+    pool:
+      name: ${{ parameters.AgentPool }}
+      os: 'linux'
 
     variables:
     - name: OnnxRuntimeBuildDirectory

tools/ci_build/github/azure-pipelines/nodejs/templates/test_win.yml

@@ -10,8 +10,9 @@ stages:
   - job: Nodejs_Test_${{ parameters.StageSuffix }}
     workspace:
       clean: all
-    timeoutInMinutes:  120
-    pool: ${{ parameters.AgentPool }}
+    timeoutInMinutes: 120
+    pool:
+      name: ${{ parameters.AgentPool }}
     variables:
     - name: OnnxRuntimeBuildDirectory
       value: '$(Build.BinariesDirectory)'

tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml

@@ -1,7 +1,7 @@
 parameters:
   AgentPool: 'onnxruntime-Ubuntu2204-AMD-CPU'
   ArtifactSuffix: ''
-  NugetPackageName : ''
+  NugetPackageName: ''
   StageSuffix: 'CPU'
   # More Suffix is used to differentiate testing for GPU and GPU-Windows/GPU-Linux packages
   MoreSuffix: ''
@@ -20,7 +20,9 @@ stages:
     workspace:
       clean: all
     timeoutInMinutes: 120
-    pool: ${{ parameters.AgentPool }}
+    pool:
+      name: ${{ parameters.AgentPool }}
+      os: 'linux'
 
     variables:
     - template: ../../templates/common-variables.yml
@@ -91,24 +93,24 @@ stages:
               "
         displayName: 'Run Package Test'
     - ${{ else }}:
-        - task: CmdLine@2
-          displayName: 'Create symlink for test models'
-          inputs:
-            script: |
-              ln -sf /data/models $(Build.BinariesDirectory)
-        - task: Bash@3
-          displayName: 'Run Package Test'
-          inputs:
-            targetType: filePath
-            filePath: '$(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh'
-            arguments: '$(Build.BinariesDirectory)/nuget-artifact $(NuGetPackageVersionNumber)'
-            workingDirectory: $(Build.BinariesDirectory)
-          env:
-            OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory)
-            DisableContribOps: $(DisableContribOps)
-            DisableMlOps: $(DisableMlOps)
-            IsReleaseBuild: $(IsReleaseBuild)
-            PACKAGENAME: ${{ parameters.NugetPackageName }}
+      - task: CmdLine@2
+        displayName: 'Create symlink for test models'
+        inputs:
+          script: |
+            ln -sf /data/models $(Build.BinariesDirectory)
+      - task: Bash@3
+        displayName: 'Run Package Test'
+        inputs:
+          targetType: filePath
+          filePath: '$(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh'
+          arguments: '$(Build.BinariesDirectory)/nuget-artifact $(NuGetPackageVersionNumber)'
+          workingDirectory: $(Build.BinariesDirectory)
+        env:
+          OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory)
+          DisableContribOps: $(DisableContribOps)
+          DisableMlOps: $(DisableMlOps)
+          IsReleaseBuild: $(IsReleaseBuild)
+          PACKAGENAME: ${{ parameters.NugetPackageName }}
 
     - template: ../../templates/component-governance-component-detection-steps.yml
       parameters:

tools/ci_build/github/azure-pipelines/stages/c-api-linux-cpu-stage.yml

@@ -0,0 +1,11 @@
+stages:
+- stage: Linux_C_API_Packaging_CPU
+  dependsOn: []
+  jobs:
+  - template: ../templates/c-api-linux-cpu.yml
+    parameters:
+      OnnxruntimeArch: 'x64'
+      OnnxruntimeNodejsBindingArch: 'x64'
+      PoolName: 'onnxruntime-Ubuntu2204-AMD-CPU'
+      PackageJava: false
+      PackageNodeJS: false

tools/ci_build/github/azure-pipelines/stages/download-java-tools-stage.yml

@@ -5,6 +5,7 @@ stages:
   - job: Download_Java_Tools
     pool:
       name: 'onnxruntime-Ubuntu2204-AMD-CPU'
+      os: linux
     steps:
     - checkout: none
     - task: CmdLine@2
@@ -17,11 +18,13 @@ stages:
           wget --tries=3 https://oss.sonatype.org/service/local/repositories/releases/content/com/google/protobuf/protobuf-java/3.21.7/protobuf-java-3.21.7.jar -P ./
           popd
         workingDirectory: '$(Agent.TempDirectory)'
-    - task: PublishPipelineArtifact@1
+
+    - task: 1ES.PublishPipelineArtifact@1
       displayName: 'Publish Pipeline Java Tools Artifact'
       inputs:
         targetPath: '$(Agent.TempDirectory)/java-tools'
         artifact: 'onnxruntime-java-tools'
+
     - template: ../templates/component-governance-component-detection-steps.yml
       parameters :
         condition : 'succeeded'