[release/8.0-staging] Transition to 1ES templates (Backport #99433)

eng/pipelines/common/global-build-job.yml

@@ -30,9 +30,10 @@ parameters:
   extraVariablesTemplates: []
   isManualCodeQLBuild: false
   preBuildSteps: []
+  templatePath: 'templates'
 
 jobs:
-- template: /eng/common/templates/job/job.yml
+- template: /eng/common/${{ parameters.templatePath }}/job/job.yml
   parameters:
     ${{ if eq(parameters.hostedOs, '') }}:
       name: ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.nameSuffix) }}
@@ -141,6 +142,7 @@ jobs:
 
       - ${{ each variable in parameters.variables }}:
         - ${{ variable }}
+
     steps:
     - ${{ if eq(parameters.osGroup, 'windows') }}:
       - template: /eng/pipelines/common/templates/disable-vsupdate-or-failfast.yml
@@ -181,7 +183,7 @@ jobs:
           path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles'
 
     - ${{ if eq(parameters.isSourceBuild, true) }}:
-      - template: /eng/common/templates/steps/source-build.yml
+      - template: /eng/common/${{ parameters.templatePath }}/steps/source-build.yml
         parameters:
           platform:
             baseOS: ${{ parameters.baseOS }}
@@ -285,14 +287,16 @@ jobs:
           displayName: Collect vslogs on exit
           condition: always()
 
-    - task: PublishBuildArtifacts@1
-      displayName: Publish Logs
-      inputs:
-        PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/'
-        PublishLocation: Container
-        ${{ if notin(parameters.osGroup, 'browser', 'wasi') }}:
-          ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }}
-        ${{ if in(parameters.osGroup, 'browser', 'wasi') }}:
-          ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }}
-      continueOnError: true
-      condition: always()
+    - template: /eng/pipelines/common/templates/publish-build-artifacts.yml
+      parameters:
+        isOfficialBuild: ${{ parameters.isOfficialBuild }}
+        displayName: Publish Logs
+        inputs:
+          PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/'
+          PublishLocation: Container
+          ${{ if notin(parameters.osGroup, 'browser', 'wasi') }}:
+            ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }}
+          ${{ if in(parameters.osGroup, 'browser', 'wasi') }}:
+            ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }}
+          continueOnError: true
+        condition: always()

eng/pipelines/common/templates/pipeline-with-resources.yml

@@ -1,113 +1,124 @@
 parameters:
   - name: stages
     type: stageList
-
-resources:
-  containers:
-    - container: linux_arm
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm
-      env:
-        ROOTFS_DIR: /crossrootfs/arm
-
-    - container: linux_armv6
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-cross-armv6-raspbian-10
-      env:
-        ROOTFS_DIR: /crossrootfs/armv6
-
-    - container: linux_arm64
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64
-      env:
-        ROOTFS_DIR: /crossrootfs/arm64
-
-    - container: linux_musl_x64
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine
-      env:
-        ROOTFS_DIR: /crossrootfs/x64
-
-    - container: linux_musl_arm
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine
-      env:
-        ROOTFS_DIR: /crossrootfs/arm
-
-    - container: linux_musl_arm64
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine
-      env:
-        ROOTFS_DIR: /crossrootfs/arm64
-
-    # This container contains all required toolsets to build for Android and for Linux with bionic libc.
-    - container: linux_bionic
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64
-
-    # This container contains all required toolsets to build for Android as well as tooling to build docker images.
-    - container: android_docker
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-android-docker
-
-    - container: linux_x64
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64
-      env:
-        ROOTFS_DIR: /crossrootfs/x64
-
-    - container: linux_x86
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-x86
-      env:
-        ROOTFS_DIR: /crossrootfs/x86
-
-    - container: linux_x64_dev_innerloop
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04
-
-    # We use a CentOS Stream 9 image here to test building from source on CentOS Stream 9.
-    - container: SourceBuild_centos_x64
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9
-
-    # AlmaLinux 8 is a RHEL 8 rebuild, so we use it to test building from source on RHEL 8.
-    - container: SourceBuild_linux_x64
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build
-
-    - container: linux_s390x
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-s390x
-      env:
-        ROOTFS_DIR: /crossrootfs/s390x
-
-    - container: linux_ppc64le
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-ppc64le
-      env:
-        ROOTFS_DIR: /crossrootfs/ppc64le
-
-    - container: linux_riscv64
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-riscv64
-      env:
-        ROOTFS_DIR: /crossrootfs/riscv64
-
-    - container: debian-12-gcc13-amd64
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-gcc13-amd64
-
-    - container: linux_x64_llvmaot
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9
-
-    - container: browser_wasm
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6
-      env:
-        ROOTFS_DIR: /crossrootfs/x64
-
-    - container: wasi_wasm
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6
-      env:
-        ROOTFS_DIR: /crossrootfs/x64
-
-    - container: freebsd_x64
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-freebsd-12
-      env:
-        ROOTFS_DIR: /crossrootfs/x64
-
-    - container: tizen_armel
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-armel-tizen
-      env:
-        ROOTFS_DIR: /crossrootfs/armel
-
-    - container: debpkg
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg
-
-    - container: rpmpkg
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm
-
-stages: ${{ parameters.stages }}
+  - name: isOfficialBuild
+    type: boolean
+    default: false
+
+extends:
+  template: templateDispatch.yml
+  parameters:
+    ${{ if parameters.isOfficialBuild }}:
+      templatePath: template1es.yml
+    ${{ else }}:
+      templatePath: templatePublic.yml
+
+    stages: ${{ parameters.stages }}
+
+    containers:
+      linux_arm:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm
+        env:
+          ROOTFS_DIR: /crossrootfs/arm
+
+      linux_armv6:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-cross-armv6-raspbian-10
+        env:
+          ROOTFS_DIR: /crossrootfs/armv6
+
+      linux_arm64:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-biarch-amd64-arm64
+        env:
+          ROOTFS_HOST_DIR: /crossrootfs/x64
+          ROOTFS_DIR: /crossrootfs/arm64
+
+      linux_musl_x64:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine
+        env:
+          ROOTFS_DIR: /crossrootfs/x64
+
+      linux_musl_arm:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine
+        env:
+          ROOTFS_DIR: /crossrootfs/arm
+
+      linux_musl_arm64:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine
+        env:
+          ROOTFS_DIR: /crossrootfs/arm64
+
+      # This container contains all required toolsets to build for Android and for Linux with bionic libc.
+      linux_bionic:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64
+
+      # This container contains all required toolsets to build for Android as well as tooling to build docker images.
+      android_docker:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-android-docker
+
+      linux_x64:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64
+        env:
+          ROOTFS_DIR: /crossrootfs/x64
+
+      linux_x86:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-x86
+        env:
+          ROOTFS_DIR: /crossrootfs/x86
+
+      linux_x64_dev_innerloop:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04
+
+      # We use a CentOS Stream 9 image here to test building from source on CentOS Stream 9.
+      SourceBuild_centos_x64:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9
+
+      # AlmaLinux 8 is a RHEL 8 rebuild, so we use it to test building from source on RHEL 8.
+      SourceBuild_linux_x64:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build
+
+      linux_s390x:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-s390x
+        env:
+          ROOTFS_DIR: /crossrootfs/s390x
+
+      linux_ppc64le:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-ppc64le
+        env:
+          ROOTFS_DIR: /crossrootfs/ppc64le
+
+      linux_riscv64:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-riscv64
+        env:
+          ROOTFS_DIR: /crossrootfs/riscv64
+
+      debian-12-gcc13-amd64:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-gcc13-amd64
+
+      linux_x64_llvmaot:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9
+
+      browser_wasm:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6
+        env:
+          ROOTFS_DIR: /crossrootfs/x64
+
+      wasi_wasm:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6
+        env:
+          ROOTFS_DIR: /crossrootfs/x64
+
+      freebsd_x64:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-freebsd-13
+        env:
+          ROOTFS_DIR: /crossrootfs/x64
+
+      tizen_armel:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-armel-tizen
+        env:
+          ROOTFS_DIR: /crossrootfs/armel
+
+      debpkg:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg
+
+      rpmpkg:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm

eng/pipelines/common/templates/publish-build-artifacts.yml

@@ -0,0 +1,22 @@
+parameters:
+  - name: isOfficialBuild
+    type: boolean
+  - name: displayName
+    type: string
+  - name: inputs
+    type: object
+  - name: condition
+    type: string
+    default: ''
+
+steps:
+  - ${{ if parameters.isOfficialBuild }}:
+    - task: 1ES.PublishBuildArtifacts@1
+      displayName: ${{ parameters.displayName }}
+      inputs: ${{ parameters.inputs }}
+      condition: ${{ parameters.condition }}
+  - ${{ else }}:
+    - task: PublishBuildArtifacts@1
+      displayName: ${{ parameters.displayName }}
+      inputs: ${{ parameters.inputs }}
+      condition: ${{ parameters.condition }}

eng/pipelines/common/templates/publish-pipeline-artifacts.yml

@@ -0,0 +1,17 @@
+parameters:
+- name: displayName
+  type: string
+- name: inputs
+  type: object
+- name: isOfficialBuild
+  type: boolean
+
+steps:
+  - ${{ if parameters.isOfficialBuild }}:
+    - task: 1ES.PublishPipelineArtifact@1
+      displayName: ${{ parameters.displayName }}
+      inputs: ${{ parameters.inputs }}
+  - ${{ else }}:
+    - task: PublishPipelineArtifact@1
+      displayName: ${{ parameters.displayName }}
+      inputs: ${{ parameters.inputs }}

eng/pipelines/common/templates/runtimes/build-test-job.yml

@@ -14,6 +14,7 @@ parameters:
   dependsOn: []
   dependOnEvaluatePaths: false
   crossBuild: false
+  isOfficialBuild: false
 
 ### Build managed test components (native components are getting built as part
 ### of the product build job).
@@ -142,12 +143,13 @@ jobs:
         artifactName: $(microsoftNetSdkIlArtifactName)
         displayName: 'Microsoft.NET.Sdk.IL package'
 
-
     # Publish Logs
-    - task: PublishPipelineArtifact@1
-      displayName: Publish Logs
-      inputs:
-        targetPath: $(Build.SourcesDirectory)/artifacts/log
-        artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_Attempt$(System.JobAttempt)_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}'
-      continueOnError: true
-      condition: always()
+    - template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml
+      parameters:
+        displayName: Publish Logs
+        isOfficialBuild: ${{ parameters.isOfficialBuild }}
+        inputs:
+          targetPath: $(Build.SourcesDirectory)/artifacts/log
+          ArtifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_Attempt$(System.JobAttempt)_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}'
+          continueOnError: true
+          condition: always()

eng/pipelines/common/templates/runtimes/xplat-job.yml

@@ -20,11 +20,12 @@ parameters:
   enableMicrobuild: ''
   gatherAssetManifests: false
   disableComponentGovernance: false
+  templatePath: 'templates'
 
   variables: {} ## any extra variables to add to the defaults defined below
 
 jobs:
-- template: /eng/common/templates/job/job.yml
+- template: /eng/common/${{ parameters.templatePath }}/job/job.yml
   parameters:
 
     name: ${{ parameters.name }}