PR-CI: Split Build and Analyze into independent LOC-based batched matrices

eng/pipelines/templates/jobs/batched-analyze.yml

@@ -0,0 +1,52 @@
+parameters:
+  - name: Artifacts
+    type: object
+    default: []
+  - name: ServiceDirectory
+    type: string
+    default: not-specified
+  - name: SDKType
+    type: string
+    default: all
+  - name: BuildSnippets
+    type: boolean
+    default: true
+  - name: ArtifactName
+    type: string
+    default: BuildPackagesArtifact
+  - name: DependsOn
+    type: string
+    default: ''
+  - name: CloudConfig
+    type: object
+    default: {}
+  - name: Matrix
+    type: string
+  - name: UsePlatformContainer
+    type: boolean
+    default: false
+  - name: OSName
+    type: string
+
+jobs:
+  - job:
+    displayName: "Analyze"
+    dependsOn: ${{ parameters.DependsOn }}
+    condition: and(succeededOrFailed(), ne(${{ parameters.Matrix }}, '{}'))
+
+    strategy:
+      matrix: $[ ${{ parameters.Matrix }} ]
+
+    pool:
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
+      os: linux
+
+    steps:
+      - template: /eng/pipelines/templates/steps/analyze.yml
+        parameters:
+          Artifacts: ${{ parameters.Artifacts }}
+          ServiceDirectory: ${{ parameters.ServiceDirectory }}
+          SDKType: ${{ parameters.SDKType }}
+          BuildSnippets: ${{ parameters.BuildSnippets }}
+          ArtifactName: ${{ parameters.ArtifactName }}

eng/pipelines/templates/jobs/batched-build-analyze.yml → eng/pipelines/templates/jobs/batched-build.yml

@@ -41,16 +41,13 @@ jobs:
       value: ${{ parameters.SDKType }}
 
     pool:
-      # 1es pipeline templates converts `image` to demands: ImageOverride under the hood
-      # which is incompatible with image selection in the default non-1es hosted pools
       ${{ if eq(parameters.OSName, 'macOS') }}:
         vmImage: $(OSVmImage)
       ${{ else }}:
         image: $(OSVmImage)
       name: $(Pool)
       os: ${{ parameters.OSName }}
     ${{ if eq(parameters.UsePlatformContainer, 'true') }}:
-      # Add a default so the job doesn't fail when the matrix is empty
       container: $[ variables['Container'] ]
 
     steps:
@@ -60,26 +57,3 @@ jobs:
           TestPipeline: ${{ parameters.TestPipeline }}
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           SDKType: ${{ parameters.SDKType }}
-
-  - job:
-    displayName: "Analyze"
-    dependsOn: ${{ parameters.DependsOn }}
-    condition: and(succeededOrFailed(), ne(${{ parameters.Matrix }}, '{}'))
-
-    # allow duplication of the job by matrix
-    strategy:
-      matrix: $[ ${{ parameters.Matrix }} ]
-
-    # but immediately ignore the matrix for the job name
-    pool:
-      name: $(LINUXPOOL)
-      image: $(LINUXVMIMAGE)
-      os: linux
-
-    steps:
-      - template: /eng/pipelines/templates/steps/analyze.yml
-        parameters:
-          Artifacts: ${{ parameters.Artifacts }}
-          ServiceDirectory: ${{ parameters.ServiceDirectory }}
-          SDKType: ${{ parameters.SDKType }}
-          BuildSnippets: ${{ parameters.BuildSnippets }}

eng/pipelines/templates/jobs/ci.yml

@@ -45,10 +45,11 @@ parameters:
 
 jobs:
   - ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
+    # Build matrix - larger LOC batches (1.2M) for fewer jobs and lower per-job overhead.
     - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml
       parameters:
         GenerateJobName: generate_build_matrix
-        JobTemplatePath: /eng/pipelines/templates/jobs/batched-build-analyze.yml
+        JobTemplatePath: /eng/pipelines/templates/jobs/batched-build.yml
         Pools: # eng/pipelines/templates/stages/build-matrix.json only contains windows OS currently so only pass the Windows pool to avoid the other OS's being marked skipped in DevOps
           - name: Windows
             filter: .*Windows.*Pool$
@@ -63,7 +64,7 @@ jobs:
           - "!SessionRecords"
         EnablePRGeneration: true
         PRMatrixSetting: ProjectNames
-        PRJobBatchSize: 5
+        PRJobBatchSize: 1
         AdditionalParameters:
           Artifacts: ${{ parameters.Artifacts }}
           TestPipeline: ${{ parameters.TestPipeline }}
@@ -77,6 +78,46 @@ jobs:
               PublishingArtifactName: BuildPackagesArtifact
               ForceDirect: true
               ExcludePaths: ${{ parameters.ExcludePaths }}
+              EnableLocWeighting: true
+              LocTarget: 1200000
+              PublishArtifact: true
+
+    # Analyze matrix - smaller LOC batches (300k) for more parallelism so analyze wall-clock matches build.
+    - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml
+      parameters:
+        GenerateJobName: generate_analyze_matrix
+        JobTemplatePath: /eng/pipelines/templates/jobs/batched-analyze.yml
+        Pools: # build-matrix.json only contains Windows entries; batched-analyze.yml overrides pool to Linux internally.
+          - name: Windows
+            filter: .*Windows.*Pool$
+            os: windows
+        MatrixConfigs:
+        - Name: NET_ci_analyze_base
+          Path: eng/pipelines/templates/stages/build-matrix.json
+          Selection: sparse
+          GenerateVMJobs: true
+        SparseCheckoutPaths:
+          - "/*"
+          - "!SessionRecords"
+        EnablePRGeneration: true
+        PRMatrixSetting: ProjectNames
+        PRJobBatchSize: 1
+        AdditionalParameters:
+          Artifacts: ${{ parameters.Artifacts }}
+          ServiceDirectory: ${{ parameters.ServiceDirectory }}
+          SDKType: ${{ parameters.SDKType }}
+          BuildSnippets: ${{ parameters.BuildSnippets }}
+          ArtifactName: AnalyzePackageInfoArtifact
+        PreGenerationSteps:
+          - template: /eng/pipelines/templates/steps/pr-matrix-presteps.yml
+            parameters:
+              ServiceDirectory: ${{ parameters.ServiceDirectory }}
+              PublishingArtifactName: AnalyzePackageInfoArtifact
+              ForceDirect: true
+              ExcludePaths: ${{ parameters.ExcludePaths }}
+              EnableLocWeighting: true
+              LocTarget: 300000
+              PublishArtifact: true
 
   - ${{ else }}:
     - job: Build

eng/pipelines/templates/steps/analyze.yml

@@ -11,6 +11,9 @@ parameters:
   - name: BuildSnippets
     type: boolean
     default: true
+  - name: ArtifactName
+    type: string
+    default: BuildPackagesArtifact
 
 steps:
   - ${{ if not(contains(variables['Build.DefinitionName'], '-pr - ')) }}:
@@ -27,7 +30,7 @@ steps:
   - ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
     - task: DownloadPipelineArtifact@2
       inputs:
-        artifact: BuildPackagesArtifact
+        artifact: ${{ parameters.ArtifactName }}
         path: $(Build.ArtifactStagingDirectory)/PackageInfo
   - ${{ else }}:
     - template: /eng/common/pipelines/templates/steps/save-package-properties.yml

eng/pipelines/templates/steps/pr-matrix-presteps.yml

@@ -9,6 +9,15 @@ parameters:
   - name: ExcludePaths
     type: object
     default: []
+  - name: EnableLocWeighting
+    type: boolean
+    default: false
+  - name: LocTarget
+    type: number
+    default: 474000
+  - name: PublishArtifact
+    type: boolean
+    default: true
 
 steps:
   - template: /eng/pipelines/templates/steps/install-dotnet.yml
@@ -28,10 +37,11 @@ steps:
         | Copy-Item -Destination "$(Build.ArtifactStagingDirectory)/PackageInfo/$($_.Name)" -Force
     displayName: Copy ArtifactInfo from Publishing Directory
 
-  - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
-    parameters:
-      ArtifactName: '${{ parameters.PublishingArtifactName }}'
-      ArtifactPath: '$(Build.ArtifactStagingDirectory)/PackageInfoPublishing'
+  - ${{ if eq(parameters.PublishArtifact, true) }}:
+    - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
+      parameters:
+        ArtifactName: '${{ parameters.PublishingArtifactName }}'
+        ArtifactPath: '$(Build.ArtifactStagingDirectory)/PackageInfoPublishing'
 
   - pwsh: |
       $packageInfos = Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/PackageInfo" -Filter "*.json" -Recurse `
@@ -62,3 +72,19 @@ steps:
     displayName: Only target direct packages for build and test
     condition: and(succeededOrFailed(), eq(${{ parameters.ForceDirect }}, true))
 
+  - ${{ if eq(parameters.EnableLocWeighting, true) }}:
+    - pwsh: |
+        eng/scripts/Get-PackageLocWeights.ps1 `
+          -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" `
+          -RepoRoot "$(Build.SourcesDirectory)" `
+          -OutputFile "$(Build.ArtifactStagingDirectory)/loc-weights.json"
+      displayName: Count LOC per package for build/analyze weighting
+      condition: and(succeededOrFailed(), eq(variables['Build.Reason'], 'PullRequest'))
+
+    - pwsh: |
+        eng/scripts/Apply-WeightedBatching.ps1 `
+          -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" `
+          -WeightsFile "$(Build.ArtifactStagingDirectory)/loc-weights.json" `
+          -Target ${{ parameters.LocTarget }}
+      displayName: Apply LOC-weighted batching (target ${{ parameters.LocTarget }})
+      condition: and(succeededOrFailed(), eq(variables['Build.Reason'], 'PullRequest'))