Update buildtools-prereqs pipelines to 1ES Pipeline templates

eng/common/templates/jobs/build-images.yml

@@ -75,9 +75,9 @@ jobs:
       }
       echo "##vso[task.setvariable variable=baseContainerRepoPath]$baseContainerRepoPath"
     displayName: Set Base Container Repo Path
-  - template: ${{ format('../steps/init-docker-{0}.yml', parameters.dockerClientOS) }}
+  - template: /eng/common/templates/jobs/${{ format('../steps/init-docker-{0}.yml', parameters.dockerClientOS) }}@self
   - ${{ parameters.customInitSteps }}
-  - template: ../steps/set-image-info-path-var.yml
+  - template: /eng/common/templates/steps/set-image-info-path-var.yml@self
     parameters:
       publicSourceBranch: $(publicSourceBranch)
   - powershell: echo "##vso[task.setvariable variable=imageBuilderBuildArgs]"
@@ -121,9 +121,13 @@ jobs:
       $(imageBuilderBuildArgs)
     name: BuildImages
     displayName: Build Images
-  - publish: $(Build.ArtifactStagingDirectory)/$(legName)-image-info.json
-    artifact: $(legName)-image-info-$(System.JobAttempt)
-    displayName: Publish Image Info File Artifact
+  - template: /eng/common/templates/steps/publish-artifact.yml@self
+    parameters:
+      path: $(Build.ArtifactStagingDirectory)/$(legName)-image-info.json
+      artifactName: $(legName)-image-info-$(System.JobAttempt)
+      displayName: Publish Image Info File Artifact
+      internalProjectName: ${{ parameters.internalProjectName }}
+      publicProjectName: ${{ parameters.publicProjectName }}
   - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
       # Define the task here to load it into the agent so that we can invoke the tool manually
       # TODO: Revert the build-specific pinned version when https://github.com/dotnet/docker-tools/issues/1152 is fixed
@@ -135,14 +139,19 @@ jobs:
     - powershell: |
         $images = "$(BuildImages.builtImages)"
         if (-not $images) { return 0 }
+
         # There can be leftover versions of the task left on the agent if it's not fresh. So find the latest version.
         $taskDir = $(Get-ChildItem -Recurse -Directory -Filter "ManifestGeneratorTask*" -Path '$(Agent.WorkFolder)')[-1].FullName
-        $manifestToolDllPath = $(Get-ChildItem -Recurse -File -Filter "Microsoft.ManifestTool.dll" -Path $taskDir).FullName
+
+        # There may be multiple version directories within the task directory. Use the latest.
+        $taskVersionDir = $(Get-ChildItem -Directory $taskDir | Sort-Object)[-1].FullName
+
+        $manifestToolDllPath = $(Get-ChildItem -Recurse -File -Filter "Microsoft.ManifestTool.dll" -Path $taskVersionDir).FullName
 
         # Check whether the manifest task installed its own version of .NET.
         # To be more robust, we'll handle varying implementations that it's had.
         # First check for a dotnet folder in the task location
-        $dotnetDir = $(Get-ChildItem -Recurse -Directory -Filter "dotnet-*" -Path $taskDir).FullName
+        $dotnetDir = $(Get-ChildItem -Recurse -Directory -Filter "dotnet-*" -Path $taskVersionDir).FullName
         if (-not $dotnetDir) {
           # If it's not there, check in the agent tools location
           $dotnetDir = $(Get-ChildItem -Recurse -Directory -Filter "*dotnet-*" -Path "$(Agent.ToolsDirectory)").FullName
@@ -177,12 +186,16 @@ jobs:
       displayName: Generate SBOMs
       condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''))
   - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-    - template: ${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}
+    - template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self
       parameters:
         condition: ne(variables.testScriptPath, '')
-  - template: ${{ format('../steps/cleanup-docker-{0}.yml', parameters.dockerClientOS) }}
+  - template: /eng/common/templates/jobs/${{ format('../steps/cleanup-docker-{0}.yml', parameters.dockerClientOS) }}@self
   - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
-    - publish: $(sbomDirectory)
-      artifact: $(legName)-sboms
-      displayName: Publish SBOM
-      condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''))
+    - template: /eng/common/templates/steps/publish-artifact.yml@self
+      parameters:
+        path: $(sbomDirectory)
+        artifactName: $(legName)-sboms
+        displayName: Publish SBOM
+        internalProjectName: ${{ parameters.internalProjectName }}
+        publicProjectName: ${{ parameters.publicProjectName }}
+        condition: ne(variables['BuildImages.builtImages'], '')

eng/common/templates/jobs/cg-build-projects.yml

@@ -0,0 +1,17 @@
+# This job builds all projects in the repository. It is intended to be used for CG purposes.
+# The 1ES CG step does not scan artifacts that are built within Dockerfiles therefore they
+# need to be built outside of Dockerfiles.
+jobs:
+- job: BuildProjects
+  displayName: Build Projects
+  pool:
+    name: NetCore1ESPool-Internal
+    image: 1es-ubuntu-2204
+    os: linux
+  steps:
+  - powershell: >
+      ./eng/common/Install-DotNetSdk.ps1 /usr/share/.dotnet
+    displayName: Run Dotnet Install Script
+  - script: >
+      find . -name '*.csproj' | grep $(cgBuildGrepArgs) | xargs -n 1 /usr/share/.dotnet/dotnet build
+    displayName: Build Projects

eng/common/templates/jobs/copy-base-images.yml

@@ -4,16 +4,16 @@ parameters:
   additionalOptions: null
   publicProjectName: null
   customInitSteps: []
-  
+
 jobs:
 - job: ${{ parameters.name }}
   pool: ${{ parameters.pool }}
   steps:
-  - template: ../steps/init-docker-linux.yml
+  - template: /eng/common/templates/steps/init-docker-linux.yml@self
   - ${{ parameters.customInitSteps }}
-  - template: ../steps/copy-base-images.yml
+  - template: /eng/common/templates/steps/copy-base-images.yml@self
     parameters:
       additionalOptions: ${{ parameters.additionalOptions }}
       publicProjectName: ${{ parameters.publicProjectName }}
       continueOnError: true
-  - template: ../steps/cleanup-docker-linux.yml
+  - template: /eng/common/templates/steps/cleanup-docker-linux.yml@self

eng/common/templates/jobs/generate-matrix.yml

@@ -10,13 +10,13 @@ jobs:
 - job: ${{ parameters.name }}
   pool: ${{ parameters.pool }}
   steps:
-  - template: ../steps/retain-build.yml
-  - template: ../steps/init-docker-linux.yml
-  - template: ../steps/validate-branch.yml
+  - template: /eng/common/templates/steps/retain-build.yml@self
+  - template: /eng/common/templates/steps/init-docker-linux.yml@self
+  - template: /eng/common/templates/steps/validate-branch.yml@self
     parameters:
       internalProjectName: ${{ parameters.internalProjectName }}
   - ${{ if eq(parameters.isTestStage, true) }}:
-    - template: ../steps/download-build-artifact.yml
+    - template: /eng/common/templates/steps/download-build-artifact.yml@self
       parameters:
         targetPath: $(Build.ArtifactStagingDirectory)
         artifactName: image-info
@@ -38,4 +38,4 @@ jobs:
       $(additionalGenerateBuildMatrixOptions)
     displayName: Generate ${{ parameters.matrixType }} Matrix
     name: matrix
-  - template: ../steps/cleanup-docker-linux.yml
+  - template: /eng/common/templates/steps/cleanup-docker-linux.yml@self

eng/common/templates/jobs/post-build.yml

@@ -1,5 +1,7 @@
 parameters:
   pool: {}
+  internalProjectName: null
+  publicProjectName: null
 
 jobs:
 - job: Build
@@ -8,8 +10,8 @@ jobs:
     imageInfosSubDir: "/image-infos"
     sbomSubDir: "/sbom"
   steps:
-  - template: ../steps/init-docker-linux.yml
-  - template: ../steps/download-build-artifact.yml
+  - template: /eng/common/templates/steps/init-docker-linux.yml@self
+  - template: /eng/common/templates/steps/download-build-artifact.yml@self
     parameters:
       targetPath: $(Build.ArtifactStagingDirectory)
   - powershell: |
@@ -23,7 +25,7 @@ jobs:
       New-Item -ItemType Directory -Path $(Build.ArtifactStagingDirectory)$(sbomSubDir)
       Get-ChildItem -Directory -Filter "*-sboms" $(Build.ArtifactStagingDirectory) |
         ForEach-Object {
-          Get-ChildItem $_ -Directory | Move-Item -Verbose -Destination $(Build.ArtifactStagingDirectory)$(sbomSubDir)
+          Get-ChildItem $_ -Directory | Move-Item -Force -Verbose -Destination $(Build.ArtifactStagingDirectory)$(sbomSubDir)
         }
     displayName: Consolidate SBOMs to Single Directory
   - powershell: |
@@ -53,9 +55,17 @@ jobs:
       $(artifactsPath)$(imageInfosSubDir)/image-info.json
       $(manifestVariables)
     displayName: Merge Image Info Files
-  - publish: $(Build.ArtifactStagingDirectory)$(sbomSubDir)
-    artifact: sboms
-    displayName: Publish SBOM Artifact
-  - publish: $(Build.ArtifactStagingDirectory)$(imageInfosSubDir)/image-info.json
-    artifact: image-info
-    displayName: Publish Image Info File Artifact
+  - template: /eng/common/templates/steps/publish-artifact.yml@self
+    parameters:
+      path: $(Build.ArtifactStagingDirectory)$(sbomSubDir)
+      artifactName: sboms
+      displayName: Publish SBOM Artifact
+      internalProjectName: ${{ parameters.internalProjectName }}
+      publicProjectName: ${{ parameters.publicProjectName }}
+  - template: /eng/common/templates/steps/publish-artifact.yml@self
+    parameters:
+      path: $(Build.ArtifactStagingDirectory)$(imageInfosSubDir)/image-info.json
+      artifactName: image-info
+      displayName: Publish Image Info File Artifact
+      internalProjectName: ${{ parameters.internalProjectName }}
+      publicProjectName: ${{ parameters.publicProjectName }}

eng/common/templates/jobs/publish.yml

@@ -23,29 +23,33 @@ jobs:
       value: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ]
   - ${{ parameters.customPublishVariables }}
   steps:
-  - template: ../steps/retain-build.yml
-  - template: ../steps/init-docker-linux.yml
+  - template: /eng/common/templates/steps/retain-build.yml@self
+  - template: /eng/common/templates/steps/init-docker-linux.yml@self
   - pwsh: |
       $azdoOrgName = Split-Path -Leaf $Env:SYSTEM_COLLECTIONURI
       echo "##vso[task.setvariable variable=azdoOrgName]$azdoOrgName"
     displayName: Set Publish Variables
   - ${{ parameters.customInitSteps }}
-  - template: ../steps/validate-branch.yml
+  - template: /eng/common/templates/steps/validate-branch.yml@self
     parameters:
       internalProjectName: ${{ parameters.internalProjectName }}
-  - template: ../steps/download-build-artifact.yml
+  - template: /eng/common/templates/steps/download-build-artifact.yml@self
     parameters:
       targetPath: $(Build.ArtifactStagingDirectory)
       artifactName: image-info
-  - template: ../steps/set-image-info-path-var.yml
+  - template: /eng/common/templates/steps/set-image-info-path-var.yml@self
     parameters:
       publicSourceBranch: $(publicSourceBranch)
-  - template: ../steps/set-dry-run.yml
+  - template: /eng/common/templates/steps/set-dry-run.yml@self
   - script: echo $(sourceBuildId) > $(Build.ArtifactStagingDirectory)/source-build-id.txt
     displayName: Write Source Build ID to File
-  - publish: $(Build.ArtifactStagingDirectory)/source-build-id.txt
-    artifact: source-build-id
-    displayName: Publish Source Build ID Artifact
+  - template: /eng/common/templates/steps/publish-artifact.yml@self
+    parameters:
+      path: $(Build.ArtifactStagingDirectory)/source-build-id.txt
+      artifactName: source-build-id
+      displayName: Publish Source Build ID Artifact
+      internalProjectName: ${{ parameters.internalProjectName }}
+      publicProjectName: ${{ parameters.publicProjectName }}
   - script: echo "##vso[task.setvariable variable=imageQueueTime]$(date --rfc-2822)"
     displayName: Set Publish Variables
   - script: >
@@ -79,16 +83,20 @@ jobs:
       $(imageBuilder.pathArgs)
       $(imageBuilder.commonCmdArgs)
     displayName: Publish Manifest
-  - publish: $(Build.ArtifactStagingDirectory)/image-info.json
-    artifact: image-info-final-$(System.JobAttempt)
-    displayName: Publish Image Info File Artifact
-  - template: ../steps/wait-for-mcr-image-ingestion.yml
+  - template: /eng/common/templates/steps/publish-artifact.yml@self
+    parameters:
+      path: $(Build.ArtifactStagingDirectory)/image-info.json
+      artifactName: image-info-final-$(System.JobAttempt)
+      displayName: Publish Image Info File Artifact
+      internalProjectName: ${{ parameters.internalProjectName }}
+      publicProjectName: ${{ parameters.publicProjectName }}
+  - template: /eng/common/templates/steps/wait-for-mcr-image-ingestion.yml@self
     parameters:
       imageInfoPath: '$(artifactsPath)/image-info.json'
       minQueueTime: $(imageQueueTime)
       dryRunArg: $(dryRunArg)
       condition: succeeded()
-  - template: ../steps/publish-readmes.yml
+  - template: /eng/common/templates/steps/publish-readmes.yml@self
     parameters:
       dryRunArg: $(dryRunArg)
       condition: and(succeeded(), eq(variables['publishReadme'], 'true'))
@@ -146,4 +154,4 @@ jobs:
       $(imageBuilder.commonCmdArgs)
     displayName: Post Publish Notification
     condition: and(always(), eq(variables['publishNotificationsEnabled'], 'true'))
-  - template: ../steps/cleanup-docker-linux.yml
+  - template: /eng/common/templates/steps/cleanup-docker-linux.yml@self

eng/common/templates/jobs/test-images-linux-client.yml

@@ -19,7 +19,7 @@ jobs:
   pool: ${{ parameters.pool }}
   timeoutInMinutes: ${{ parameters.testJobTimeout }}
   steps:
-  - template: ../steps/test-images-linux-client.yml
+  - template: /eng/common/templates/steps/test-images-linux-client.yml@self
     parameters:
       preBuildValidation: ${{ parameters.preBuildValidation }}
       internalProjectName: ${{ parameters.internalProjectName }}

eng/common/templates/jobs/test-images-windows-client.yml

@@ -15,7 +15,7 @@ jobs:
     matrix: $[ ${{ parameters.matrix }} ]
   timeoutInMinutes: ${{ parameters.testJobTimeout }}
   steps:
-  - template: ../steps/test-images-windows-client.yml
+  - template: /eng/common/templates/steps/test-images-windows-client.yml@self
     parameters:
       internalProjectName: ${{ parameters.internalProjectName }}
       customInitSteps: ${{ parameters.customInitSteps }}