Split Build job into Build Packages and Build rest of artifacts

eng/pipelines/templates/jobs/ci.yml

@@ -82,7 +82,33 @@ jobs:
       parameters:
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
-    - template: ../steps/build-artifacts.yml
+    - template: ../steps/build-package-artifacts.yml
+      parameters:
+        ServiceDirectory: ${{ parameters.ServiceDirectory }}
+        BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
+        TestPipeline: ${{ parameters.TestPipeline }}
+        Artifacts: ${{ parameters.Artifacts }}
+
+  - job: 'Build_Extended'
+    displayName: Build Extended
+    dependsOn: 
+      - 'Build'
+    timeoutInMinutes: 90
+    variables:
+      Codeql.Enabled: true
+      Codeql.BuildIdentifier: ${{ parameters.ServiceDirectory }}
+      Codeql.SkipTaskAutoInjection: false
+
+    pool:
+      name: azsdk-pool-mms-ubuntu-2004-general
+      vmImage: MMSUbuntu20.04
+
+    steps:
+    - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+      parameters:
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
+
+    - template: ../steps/build-extended-artifacts.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
@@ -102,7 +128,7 @@ jobs:
     timeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
 
     dependsOn:
-      - 'Build'
+      - 'Build_Extended'
 
     pool:
       name: azsdk-pool-mms-ubuntu-2004-general

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -120,7 +120,7 @@ stages:
         Artifacts: ${{ parameters.Artifacts }}
         ${{ if eq(parameters.ServiceDirectory, 'template') }}:
           TestPipeline: true
-        ArtifactName: packages
+        ArtifactName: packages_extended
         DocArtifact: documentation
         TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }}
         TargetDocRepoName: ${{ parameters.TargetDocRepoName }}

eng/pipelines/templates/steps/analyze.yml

@@ -113,7 +113,7 @@ steps:
   - task: DownloadPipelineArtifact@2
     condition: succeededOrFailed()
     inputs:
-      artifactName: 'packages'
+      artifactName: 'packages_extended'
       targetPath: $(Build.ArtifactStagingDirectory)
 
   - template: ../steps/run_bandit.yml
@@ -137,10 +137,12 @@ steps:
     parameters:
       Artifacts: ${{ parameters.Artifacts }}
       GenerateApiReviewForManualOnly: ${{ parameters.GenerateApiReviewForManualOnly }}
+      ArtifactName: "packages_extended"
 
   - template: /eng/common/pipelines/templates/steps/detect-api-changes.yml
     parameters:
       Artifacts: ${{ parameters.Artifacts }}
+      ArtifactName: "packages_extended"
 
   - template: /eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml
 

eng/pipelines/templates/steps/build-extended-artifacts.yml

@@ -0,0 +1,105 @@
+parameters:
+  - name: BeforePublishSteps
+    type: object
+    default: []
+  - name: TestPipeline
+    type: boolean
+    default: false
+  - name: ServiceDirectory
+    type: string
+    default: ''
+  - name: BuildDocs
+    type: boolean
+    default: true
+  - name: Artifacts
+    type: object
+    default: []
+  - name: DevFeedName
+    type: string
+    default: 'public/azure-sdk-for-python'
+# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files:
+#     eng/pipelines/templates/jobs/ci.tests.yml
+#     eng/pipelines/templates/jobs/ci.yml
+#     eng/pipelines/templates/jobs/live.test.yml
+
+# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`.
+steps:
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      artifactName: 'packages'
+      targetPath: $(Build.ArtifactStagingDirectory)
+
+  - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
+    parameters:
+      PackageName: "azure-template"
+      ServiceDirectory: "template"
+      TestPipeline: ${{ parameters.TestPipeline }}
+
+  - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+
+  - script: |
+      echo "##vso[build.addbuildtag]Scheduled"
+    displayName: 'Tag scheduled builds'
+    condition: and(eq(variables['Build.SourceBranchName'], variables['DefaultBranch']), eq(variables['Build.Reason'],'Schedule'))
+
+  - task: UsePythonVersion@0
+    displayName: 'Use Python $(PythonVersion)'
+    inputs:
+      versionSpec: $(PythonVersion)
+
+  - script: |
+      python -m pip install setuptools==58.3.0
+      python -m pip install -r eng/ci_tools.txt
+    displayName: 'Prep Environment'
+
+  - template: set-dev-build.yml
+    parameters:
+      ServiceDirectory: ${{ parameters.ServiceDirectory }}
+
+  - ${{if eq(variables['System.TeamProject'], 'internal') }}:
+    - template: auth-dev-feed.yml
+      parameters:
+        DevFeedName: ${{ parameters.DevFeedName }}
+
+  - task: PythonScript@0
+    displayName: 'Generate Docs'
+    condition: and(succeededOrFailed(), ${{parameters.BuildDocs}})
+    inputs:
+      scriptPath: 'scripts/devops_tasks/setup_execute_tests.py'
+      arguments: >-
+        "$(TargetingString)" 
+        --service="${{ parameters.ServiceDirectory }}" 
+        --toxenv=sphinx
+
+  - pwsh: |
+      Write-Host "##vso[task.setvariable variable=PIP_INDEX_URL]https://pypi.python.org/simple"
+    displayName: Reset PIP Index For APIStubGen
+
+  - template: /eng/pipelines/templates/steps/run_apistub.yml
+    parameters:
+      ServiceDirectory: ${{ parameters.ServiceDirectory }}
+
+  - ${{ parameters.BeforePublishSteps }}
+
+  - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
+    parameters:
+      ArtifactPath: '$(Build.ArtifactStagingDirectory)'
+      ArtifactName: 'packages_extended'
+
+  - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
+    parameters:
+      ArtifactPath: '$(Build.SourcesDirectory)/_docs'
+      CustomCondition: ${{ parameters.BuildDocs }}
+      ArtifactName: 'documentation'
+
+  - ${{if eq(variables['System.TeamProject'], 'internal') }}:
+    - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
+      displayName: 'Generate BOM'
+      condition: succeededOrFailed()
+      inputs:
+        BuildDropPath: $(Build.ArtifactStagingDirectory)
+
+    - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
+      parameters:
+        ArtifactPath: '$(Build.ArtifactStagingDirectory)/_manifest'
+        ArtifactName: 'extended_manifest'

eng/pipelines/templates/steps/build-artifacts.yml → eng/pipelines/templates/steps/build-package-artifacts.yml

@@ -8,9 +8,6 @@ parameters:
   - name: ServiceDirectory
     type: string
     default: ''
-  - name: BuildDocs
-    type: boolean
-    default: true
   - name: Artifacts
     type: object
     default: []
@@ -47,7 +44,7 @@ steps:
       python -m pip install -r eng/ci_tools.txt
     displayName: 'Prep Environment'
 
-  - template: ../steps/set-dev-build.yml
+  - template: set-dev-build.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
 
@@ -74,41 +71,17 @@ steps:
     displayName: 'Verify Readme'
 
   - ${{if eq(variables['System.TeamProject'], 'internal') }}:
-    - template: ../steps/auth-dev-feed.yml
+    - template: auth-dev-feed.yml
       parameters:
         DevFeedName: ${{ parameters.DevFeedName }}
 
-  - task: PythonScript@0
-    displayName: 'Generate Docs'
-    condition: and(succeededOrFailed(), ${{parameters.BuildDocs}})
-    inputs:
-      scriptPath: 'scripts/devops_tasks/setup_execute_tests.py'
-      arguments: >-
-        "$(TargetingString)" 
-        --service="${{ parameters.ServiceDirectory }}" 
-        --toxenv=sphinx
-
-  - pwsh: |
-      Write-Host "##vso[task.setvariable variable=PIP_INDEX_URL]https://pypi.python.org/simple"
-    displayName: Reset PIP Index For APIStubGen
-
-  - template: /eng/pipelines/templates/steps/run_apistub.yml
-    parameters:
-      ServiceDirectory: ${{ parameters.ServiceDirectory }}
-
   - ${{ parameters.BeforePublishSteps }}
 
   - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
     parameters:
       ArtifactPath: '$(Build.ArtifactStagingDirectory)'
       ArtifactName: 'packages'
 
-  - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
-    parameters:
-      ArtifactPath: '$(Build.SourcesDirectory)/_docs'
-      CustomCondition: ${{ parameters.BuildDocs }}
-      ArtifactName: 'documentation'
-
   - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
     parameters:
       ArtifactPath: '$(Build.SourcesDirectory)/.logs'
@@ -124,4 +97,4 @@ steps:
     - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
       parameters:
         ArtifactPath: '$(Build.ArtifactStagingDirectory)/_manifest'
-        ArtifactName: 'manifest'
+        ArtifactName: 'packages_manifest'

eng/pipelines/templates/steps/create-apireview.yml

@@ -5,6 +5,7 @@ parameters:
   Language: 'Python'
   APIViewUri: 'https://apiview.dev/AutoReview/CreateApiReview'
   GenerateApiReviewForManualOnly: false
+  ArtifactName: 'packages_extended'
 
 steps:
   # ideally this should be done as initial step of a job in caller template
@@ -28,6 +29,7 @@ steps:
           -BuildId $(Build.BuildId)
           -RepoName $(Build.Repository.Name)
           -Language ${{ parameters.Language }}
+          -ArtifactName ${{ parameters.ArtifactName }}
         pwsh: true
         workingDirectory: $(Pipeline.Workspace)
       displayName: Create Automatic API Review

eng/scripts/Create-ApiReview.ps1

@@ -19,7 +19,9 @@ Param (
   [Parameter(Mandatory=$True)]
   [string] $repoName,
   [Parameter(Mandatory=$True)]
-  [string] $Language
+  [string] $Language,
+  [Parameter(Mandatory=$False)]
+  [string] $ArtifactName = "packages"
 )
 
 Set-StrictMode -Version 3
@@ -28,7 +30,7 @@ Set-StrictMode -Version 3
 # Submit API review request and return status whether current revision is approved or pending or failed to create review
 function Submit-APIReview($packageArtifactname, $apiLabel, $releaseStatus, $reviewFileName)
 {
-    $params = "buildId=$buildId&artifactName=packages&originalFilePath=$packageArtifactname&reviewFilePath=$reviewFileName"
+    $params = "buildId=$buildId&artifactName=$ArtifactName&originalFilePath=$packageArtifactname&reviewFilePath=$reviewFileName"
     $params += "&label=$apiLabel&repoName=$repoName&packageName=$PackageName&project=internal"
     $uri = "$($APIViewUri)?$params"