1ES Template Conversion

eng/CredScanSuppression.json

@@ -36,6 +36,7 @@
                 "sdk/confidentialledger/azure-confidentialledger/tests/_shared/constants.py",
                 "sdk/keyvault/azure-keyvault-certificates/tests/ca.key",
                 "sdk/identity/azure-identity/tests/certificate.pfx",
+                "sdk/identity/azure-identity/tests/certificate.pem",
                 "sdk/identity/azure-identity/tests/certificate-with-password.pfx",
                 "sdk/identity/azure-identity/tests/credscan_ignore.py",
                 "sdk/identity/azure-identity/tests/ec-certificate.pem",

eng/pipelines/templates/jobs/build-conda-dependencies.yml

@@ -9,7 +9,8 @@ jobs:
   timeoutInMinutes: 90
   pool:
     name: azsdk-pool-mms-win-2022-general
-    vmImage: MMS2022
+    image: azsdk-pool-mms-win-2022-1espt
+    os: windows
   variables:
     VS_INSTALLER_URL: "https://aka.ms/vs/17/release/vs_enterprise.exe"
     VS_INSTALLER_PATH: "$(Build.BinariesDirectory)/vs_enterprise.exe"
@@ -62,7 +63,7 @@ jobs:
         sdk_build_conda -c %arguments%
       displayName: Assemble Conda Packages
 
-    - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
+    - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
       parameters:
         ArtifactPath: '$(Build.SourcesDirectory)/conda/assembled'
         ArtifactName: 'windows_distributions'
@@ -74,7 +75,7 @@ jobs:
         inputs:
           BuildDropPath: '$(Build.SourcesDirectory)/conda/output'
 
-    - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
+    - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
       parameters:
         ArtifactPath: '$(Build.SourcesDirectory)/conda/output'
         ArtifactName: 'windows_conda'
@@ -84,7 +85,9 @@ jobs:
   timeoutInMinutes: 90
   pool:
     name: azsdk-pool-mms-ubuntu-2004-general
-    vmImage: MMSUbuntu20.04
+    image: azsdk-pool-mms-ubuntu-2004-1espt
+    os: linux
+
   steps:
     - bash: |
         sudo apt-get install -y build-essential
@@ -101,6 +104,7 @@ jobs:
   pool:
     name: Azure Pipelines
     vmImage: macos-11
+    os: macOS
   variables:
     MacOSXDeploymentTarget: '10.9'
     OpenSSLDir: $(Agent.BuildDirectory)/openssl-macosx$(MacOSXDeploymentTarget)

eng/pipelines/templates/jobs/ci.tests.yml

@@ -29,11 +29,11 @@ parameters:
   - name: InjectedPackages
     type: string
     default: ''
-  - name: Matrix
-    type: string
   - name: DependsOn
+    type: object
+    default: []
+  - name: Matrix
     type: string
-    default: ''
   - name: UsePlatformContainer
     type: boolean
     default: false
@@ -43,6 +43,8 @@ parameters:
   - name: TestProxy
     type: boolean
     default: false
+  - name: OSName
+    type: string
 
 jobs:
   - job:
@@ -56,32 +58,35 @@ jobs:
     timeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
 
     dependsOn:
-       - ${{ parameters.DependsOn }}
+      - ${{ parameters.DependsOn }}
 
     strategy:
       matrix: $[ ${{ parameters.Matrix }} ]
 
     pool:
       name: $(Pool)
-      vmImage: $(OSVmImage)
-
-    ${{ if eq(parameters.UsePlatformContainer, 'true') }}:
-      # Add a default so the job doesn't fail when the matrix is empty
-      container: $[ variables['Container'] ]
+      # 1es pipeline templates converts `image` to demands: ImageOverride under the hood
+      # which is incompatible with image selection in the default non-1es hosted pools
+      ${{ if eq(parameters.OSName, 'macOS') }}:
+        vmImage: $(OSVmImage)
+      ${{ else }}:
+        image: $(OSVmImage)
+      os: ${{ parameters.OSName }}
 
     variables:
     - template: ../variables/globals.yml
     - name: InjectedPackages
       value: ${{ parameters.InjectedPackages }}
 
+
     steps:
     - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
       parameters:
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
     - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
       parameters:
-        AgentImage: $(OSVmImage)
+        AgentImage: ${{ parameters.OSName }}
 
     - task: UsePythonVersion@0
       inputs:
@@ -111,7 +116,6 @@ jobs:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         TestMarkArgument: ${{ parameters.TestMarkArgument }}
         AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"'
-        OSVmImage: $(OSVmImage)
         CoverageArg: $(CoverageArg)
         PythonVersion: $(PythonVersion)
         ToxTestEnv: $(toxenv)
@@ -127,7 +131,7 @@ jobs:
           - template: ../steps/set-dev-build.yml
             parameters:
               ServiceDirectory: ${{ parameters.ServiceDirectory }}
-          
+
           - ${{ each step in parameters.BeforeTestSteps }}:
             -  ${{ step }}
         AfterTestSteps: ${{ parameters.AfterTestSteps }}

eng/pipelines/templates/jobs/ci.yml

@@ -66,72 +66,104 @@ parameters:
     default: false
 
 jobs:
-  
+
   - ${{ if eq(parameters['AdvancedBuild'], false) }}:
     - job: 'Build'
       timeoutInMinutes: 90
 
       pool:
         name: azsdk-pool-mms-ubuntu-2004-general
-        vmImage: MMSUbuntu20.04
+        image: azsdk-pool-mms-ubuntu-2004-1espt
+        os: linux
 
       steps:
-      - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+      - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml@self
         parameters:
           BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
-      - template: ../steps/build-package-artifacts.yml
+      - template: ../steps/build-package-artifacts.yml@self
         parameters:
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
           TestPipeline: ${{ parameters.TestPipeline }}
           Artifacts: ${{ parameters.Artifacts }}
 
   - ${{ if eq(parameters['AdvancedBuild'], true) }}:
-    - job: 'Build'
+    - job: 'Build_Linux'
       timeoutInMinutes: 90
 
-      strategy:
-        matrix:
-          Linux:
-            imageName: 'MMSUbuntu22.04'
-            poolName: 'azsdk-pool-mms-ubuntu-2204-general'
-            ArtifactName: 'linux'
-          Windows:
-            imageName: 'MMS2022'
-            poolName: 'azsdk-pool-mms-win-2022-general'
-            ArtifactName: 'windows'
-          Mac:
-            imageName: 'macos-11'
-            poolName: 'Azure Pipelines'
-            ArtifactName: 'mac'
+      pool:
+        name: azsdk-pool-mms-ubuntu-2004-general
+        image: azsdk-pool-mms-ubuntu-2004-1espt
+        os: linux
+
+      steps:
+      - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml@self
+        parameters:
+          BuildTargetingString: ${{ parameters.BuildTargetingString }}
+
+      - template: ../steps/build-package-artifacts.yml@self
+        parameters:
+          ServiceDirectory: ${{ parameters.ServiceDirectory }}
+          BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
+          TestPipeline: ${{ parameters.TestPipeline }}
+          Artifacts: ${{ parameters.Artifacts }}
+          ArtifactSuffix: linux
+
+    - job: 'Build_Windows'
+      timeoutInMinutes: 90
 
       pool:
-        name: $(poolName)
-        vmImage: $(imageName)
+        name: azsdk-pool-mms-win-2022-general
+        image: azsdk-pool-mms-win-2022-1espt
+        os: windows
 
       steps:
-      - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+      - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml@self
         parameters:
           BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
-      - template: ../steps/build-package-artifacts.yml
+      - template: ../steps/build-package-artifacts.yml@self
         parameters:
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
           TestPipeline: ${{ parameters.TestPipeline }}
           Artifacts: ${{ parameters.Artifacts }}
-          ArtifactSuffix: $(ArtifactName)
+          ArtifactSuffix: windows
+
+    - job: 'Build_MacOS'
+      timeoutInMinutes: 90
+
+      pool:
+        name: 'Azure Pipelines'
+        vmImage: macos-11
+        os: macOS
+
+      steps:
+      - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml@self
+        parameters:
+          BuildTargetingString: ${{ parameters.BuildTargetingString }}
+
+      - template: ../steps/build-package-artifacts.yml@self
+        parameters:
+          ServiceDirectory: ${{ parameters.ServiceDirectory }}
+          BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
+          TestPipeline: ${{ parameters.TestPipeline }}
+          Artifacts: ${{ parameters.Artifacts }}
+          ArtifactSuffix: mac
 
     - job: 'CoalesceBuildArtifacts'
       displayName: Combine Built Artifacts
-      dependsOn: 
-        - 'Build'
+      dependsOn:
+        - 'Build_Linux'
+        - 'Build_Windows'
+        - 'Build_MacOS'
       timeoutInMinutes: 90
 
       pool:
         name: azsdk-pool-mms-ubuntu-2004-general
-        vmImage: MMSUbuntu20.04
+        image: azsdk-pool-mms-ubuntu-2004-1espt
+        os: linux
 
       steps:
       - task: DownloadPipelineArtifact@2
@@ -149,22 +181,25 @@ jobs:
           artifactName: 'packages_linux'
           targetPath: $(Build.ArtifactStagingDirectory)/packages
 
-      - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
+      - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
         parameters:
           ArtifactPath: '$(Build.ArtifactStagingDirectory)/packages'
           ArtifactName: 'packages'
 
   - job: 'Build_Extended'
     displayName: Build Extended
-    dependsOn: 
-      - 'Build'
+    dependsOn:
       - ${{ if eq(parameters['AdvancedBuild'], true) }}:
         - 'CoalesceBuildArtifacts'
+      - ${{ else }}:
+        - 'Build'
+
     timeoutInMinutes: 90
 
     pool:
       name: azsdk-pool-mms-ubuntu-2004-general
-      vmImage: MMSUbuntu20.04
+      image: azsdk-pool-mms-ubuntu-2004-1espt
+      os: linux
 
     steps:
     - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
@@ -189,7 +224,8 @@ jobs:
 
     pool:
       name: azsdk-pool-mms-ubuntu-2004-general
-      vmImage: MMSUbuntu20.04
+      image: azsdk-pool-mms-ubuntu-2004-1espt
+      os: linux
 
     steps:
     - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
@@ -220,35 +256,16 @@ jobs:
         ValidateFormatting: ${{ parameters.ValidateFormatting }}
         GenerateApiReviewForManualOnly: ${{ parameters.GenerateApiReviewForManualOnly }}
 
-  - job: Compliance
-    pool:
-      name: azsdk-pool-mms-win-2022-general
-      vmImage: MMS2022
-
-    variables:
-      Codeql.SkipTaskAutoInjection: false
-      Codeql.Enabled: true
-      Codeql.Language: python
-      Codeql.BuildIdentifier: "${{ parameters.ServiceDirectory }}"
-      Codeql.SourceRoot: "sdk/${{ parameters.ServiceDirectory }}"
-
-    # per the guidance of the codeql team:
-    # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/snippets/codeql-3000-other-issues#timeouts
-    timeoutInMinutes: 360
-
-    steps:
-      - template: /eng/common/pipelines/templates/steps/credscan.yml
-        parameters:
-          ServiceDirectory: ${{ parameters.ServiceDirectory }}
-          BaselineFilePath: $(Build.SourcesDirectory)\eng\python.gdnbaselines
-
-  - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
+  - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml
     parameters:
       JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml
+      OsVmImage: azsdk-pool-mms-ubuntu-2004-1espt
+      Pool: azsdk-pool-mms-ubuntu-2004-general
       DependsOn:
-        - 'Build'
         - ${{ if eq(parameters['AdvancedBuild'], true) }}:
           - 'CoalesceBuildArtifacts'
+        - ${{ else }}:
+          - 'Build'
       MatrixConfigs: ${{ parameters.MatrixConfigs }}
       MatrixFilters: ${{ parameters.MatrixFilters }}
       MatrixReplace: ${{ parameters.MatrixReplace }}
@@ -267,9 +284,11 @@ jobs:
         UnsupportedToxEnvironments: ${{ parameters.UnsupportedToxEnvironments }}
         TestProxy: ${{ parameters.TestProxy }}
 
-  - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
+  - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml
     parameters:
       JobTemplatePath: /eng/pipelines/templates/jobs/regression.yml
+      OsVmImage: azsdk-pool-mms-ubuntu-2004-1espt
+      Pool: azsdk-pool-mms-ubuntu-2004-general
       GenerateJobName: generate_regression_matrix
       SparseCheckoutPaths: [ "scripts/", "sdk/", "tools/azure-sdk-tools/" ]
       MatrixConfigs:
@@ -295,9 +314,10 @@ jobs:
       CloudConfig:
         Cloud: Public
       DependsOn:
-        - 'Build'
         - ${{ if eq(parameters['AdvancedBuild'], true) }}:
           - 'CoalesceBuildArtifacts'
+        - ${{ else }}:
+          - 'Build'
       AdditionalParameters:
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
         ServiceDirectory: ${{ parameters.ServiceDirectory }}