[DO NOT MERGE] CI measurement: 100pkg - Set A

eng/pipelines/templates/jobs/batched-analyze.yml

@@ -0,0 +1,48 @@
+parameters:
+  - name: Artifacts
+    type: object
+    default: []
+  - name: ServiceDirectory
+    type: string
+    default: not-specified
+  - name: SDKType
+    type: string
+    default: all
+  - name: BuildSnippets
+    type: boolean
+    default: true
+  - name: DependsOn
+    type: string
+    default: ''
+  - name: CloudConfig
+    type: object
+    default: {}
+  - name: Matrix
+    type: string
+  - name: UsePlatformContainer
+    type: boolean
+    default: false
+  - name: OSName
+    type: string
+
+jobs:
+  - job:
+    displayName: "Analyze"
+    dependsOn: ${{ parameters.DependsOn }}
+    condition: and(succeededOrFailed(), ne(${{ parameters.Matrix }}, '{}'))
+
+    strategy:
+      matrix: $[ ${{ parameters.Matrix }} ]
+
+    pool:
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
+      os: linux
+
+    steps:
+      - template: /eng/pipelines/templates/steps/analyze.yml
+        parameters:
+          Artifacts: ${{ parameters.Artifacts }}
+          ServiceDirectory: ${{ parameters.ServiceDirectory }}
+          SDKType: ${{ parameters.SDKType }}
+          BuildSnippets: ${{ parameters.BuildSnippets }}

eng/pipelines/templates/jobs/batched-build-analyze.yml → eng/pipelines/templates/jobs/batched-build.yml

@@ -41,16 +41,13 @@ jobs:
       value: ${{ parameters.SDKType }}
 
     pool:
-      # 1es pipeline templates converts `image` to demands: ImageOverride under the hood
-      # which is incompatible with image selection in the default non-1es hosted pools
       ${{ if eq(parameters.OSName, 'macOS') }}:
         vmImage: $(OSVmImage)
       ${{ else }}:
         image: $(OSVmImage)
       name: $(Pool)
       os: ${{ parameters.OSName }}
     ${{ if eq(parameters.UsePlatformContainer, 'true') }}:
-      # Add a default so the job doesn't fail when the matrix is empty
       container: $[ variables['Container'] ]
 
     steps:
@@ -60,26 +57,3 @@ jobs:
           TestPipeline: ${{ parameters.TestPipeline }}
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           SDKType: ${{ parameters.SDKType }}
-
-  - job:
-    displayName: "Analyze"
-    dependsOn: ${{ parameters.DependsOn }}
-    condition: and(succeededOrFailed(), ne(${{ parameters.Matrix }}, '{}'))
-
-    # allow duplication of the job by matrix
-    strategy:
-      matrix: $[ ${{ parameters.Matrix }} ]
-
-    # but immediately ignore the matrix for the job name
-    pool:
-      name: $(LINUXPOOL)
-      image: $(LINUXVMIMAGE)
-      os: linux
-
-    steps:
-      - template: /eng/pipelines/templates/steps/analyze.yml
-        parameters:
-          Artifacts: ${{ parameters.Artifacts }}
-          ServiceDirectory: ${{ parameters.ServiceDirectory }}
-          SDKType: ${{ parameters.SDKType }}
-          BuildSnippets: ${{ parameters.BuildSnippets }}

eng/pipelines/templates/jobs/ci.yml

@@ -45,10 +45,11 @@ parameters:
 
 jobs:
   - ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
+    # Build matrix - larger LOC batches (1.2M) for fewer jobs and lower per-job overhead.
     - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml
       parameters:
         GenerateJobName: generate_build_matrix
-        JobTemplatePath: /eng/pipelines/templates/jobs/batched-build-analyze.yml
+        JobTemplatePath: /eng/pipelines/templates/jobs/batched-build.yml
         Pools: # eng/pipelines/templates/stages/build-matrix.json only contains windows OS currently so only pass the Windows pool to avoid the other OS's being marked skipped in DevOps
           - name: Windows
             filter: .*Windows.*Pool$
@@ -63,7 +64,7 @@ jobs:
           - "!SessionRecords"
         EnablePRGeneration: true
         PRMatrixSetting: ProjectNames
-        PRJobBatchSize: 5
+        PRJobBatchSize: 1
         AdditionalParameters:
           Artifacts: ${{ parameters.Artifacts }}
           TestPipeline: ${{ parameters.TestPipeline }}
@@ -77,6 +78,45 @@ jobs:
               PublishingArtifactName: BuildPackagesArtifact
               ForceDirect: true
               ExcludePaths: ${{ parameters.ExcludePaths }}
+              EnableBuildWeighting: true
+              LocTargetSeconds: 1200000
+              PublishArtifact: true
+
+    # Analyze matrix - smaller LOC batches (300k) for more parallelism so analyze wall-clock matches build.
+    - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml
+      parameters:
+        GenerateJobName: generate_analyze_matrix
+        JobTemplatePath: /eng/pipelines/templates/jobs/batched-analyze.yml
+        Pools: # build-matrix.json only contains Windows entries; batched-analyze.yml overrides pool to Linux internally.
+          - name: Windows
+            filter: .*Windows.*Pool$
+            os: windows
+        MatrixConfigs:
+        - Name: NET_ci_analyze_base
+          Path: eng/pipelines/templates/stages/build-matrix.json
+          Selection: sparse
+          GenerateVMJobs: true
+        SparseCheckoutPaths:
+          - "/*"
+          - "!SessionRecords"
+        EnablePRGeneration: true
+        PRMatrixSetting: ProjectNames
+        PRJobBatchSize: 1
+        AdditionalParameters:
+          Artifacts: ${{ parameters.Artifacts }}
+          ServiceDirectory: ${{ parameters.ServiceDirectory }}
+          SDKType: ${{ parameters.SDKType }}
+          BuildSnippets: ${{ parameters.BuildSnippets }}
+        PreGenerationSteps:
+          - template: /eng/pipelines/templates/steps/pr-matrix-presteps.yml
+            parameters:
+              ServiceDirectory: ${{ parameters.ServiceDirectory }}
+              PublishingArtifactName: BuildPackagesArtifact
+              ForceDirect: true
+              ExcludePaths: ${{ parameters.ExcludePaths }}
+              EnableBuildWeighting: true
+              LocTargetSeconds: 300000
+              PublishArtifact: false
 
   - ${{ else }}:
     - job: Build

eng/pipelines/templates/steps/pr-matrix-presteps.yml

@@ -9,6 +9,15 @@ parameters:
   - name: ExcludePaths
     type: object
     default: []
+  - name: EnableBuildWeighting
+    type: boolean
+    default: false
+  - name: LocTargetSeconds
+    type: number
+    default: 474000
+  - name: PublishArtifact
+    type: boolean
+    default: true
 
 steps:
   - template: /eng/pipelines/templates/steps/install-dotnet.yml
@@ -28,10 +37,11 @@ steps:
         | Copy-Item -Destination "$(Build.ArtifactStagingDirectory)/PackageInfo/$($_.Name)" -Force
     displayName: Copy ArtifactInfo from Publishing Directory
 
-  - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
-    parameters:
-      ArtifactName: '${{ parameters.PublishingArtifactName }}'
-      ArtifactPath: '$(Build.ArtifactStagingDirectory)/PackageInfoPublishing'
+  - ${{ if eq(parameters.PublishArtifact, true) }}:
+    - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
+      parameters:
+        ArtifactName: '${{ parameters.PublishingArtifactName }}'
+        ArtifactPath: '$(Build.ArtifactStagingDirectory)/PackageInfoPublishing'
 
   - pwsh: |
       $packageInfos = Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/PackageInfo" -Filter "*.json" -Recurse `
@@ -62,3 +72,19 @@ steps:
     displayName: Only target direct packages for build and test
     condition: and(succeededOrFailed(), eq(${{ parameters.ForceDirect }}, true))
 
+  - ${{ if eq(parameters.EnableBuildWeighting, true) }}:
+    - pwsh: |
+        eng/scripts/Get-PackageLocWeights.ps1 `
+          -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" `
+          -RepoRoot "$(Build.SourcesDirectory)" `
+          -OutputFile "$(Build.ArtifactStagingDirectory)/loc-weights.json"
+      displayName: Count LOC per package for build/analyze weighting
+      condition: and(succeededOrFailed(), eq(variables['Build.Reason'], 'PullRequest'))
+
+    - pwsh: |
+        eng/scripts/Apply-WeightedBatching.ps1 `
+          -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" `
+          -WeightsFile "$(Build.ArtifactStagingDirectory)/loc-weights.json" `
+          -TargetSeconds ${{ parameters.LocTargetSeconds }}
+      displayName: Apply LOC-weighted batching (target ${{ parameters.LocTargetSeconds }})
+      condition: and(succeededOrFailed(), eq(variables['Build.Reason'], 'PullRequest'))

eng/scripts/Apply-WeightedBatching.ps1

@@ -0,0 +1,183 @@
+<#
+.SYNOPSIS
+Applies weighted LPT bin-packing to PackageInfo files for balanced CI test batching.
+
+.DESCRIPTION
+Reads PackageInfo JSON files and a test weights file, performs LPT (Longest Processing Time)
+bin-packing to create balanced batches, then consolidates the PackageInfo files so that each
+batch becomes a single representative file. The ArtifactName of each consolidated file contains
+the comma-separated names of all packages in that batch, which flows through to the
+ProjectNames parameter in the matrix generation.
+
+After this script runs, set PRJobBatchSize to 1 so each consolidated file becomes its own job.
+
+.PARAMETER PackageInfoFolder
+Path to the folder containing PackageInfo JSON files.
+
+.PARAMETER WeightsFile
+Path to the JSON weights file (package name → seconds).
+
+.PARAMETER TargetSeconds
+Target maximum time per bucket in seconds for direct packages. Default is 1800 (30 minutes).
+
+.PARAMETER IndirectTargetSeconds
+Target maximum time per bucket in seconds for indirect packages. Defaults to TargetSeconds if not specified.
+Indirect packages only run on Linux, so they can use a higher target than direct packages which run across all platforms.
+
+.PARAMETER DefaultWeight
+Weight assigned to packages not found in the weights file. Default is 1.
+#>
+
+[CmdletBinding()]
+param (
+  [Parameter(Mandatory = $true)][string]$PackageInfoFolder,
+  [Parameter(Mandatory = $true)][string]$WeightsFile,
+  [Parameter()][int]$TargetSeconds = 1800,
+  [Parameter()][int]$IndirectTargetSeconds = 0,
+  [Parameter()][int]$DefaultWeight = 1
+)
+
+Set-StrictMode -Version 4
+
+if ($IndirectTargetSeconds -le 0) { $IndirectTargetSeconds = $TargetSeconds }
+
+# Load weights
+$weights = @{}
+if (Test-Path $WeightsFile) {
+  try {
+    $content = Get-Content $WeightsFile -Raw | ConvertFrom-Json
+    $content.PSObject.Properties | ForEach-Object { $weights[$_.Name] = [int]$_.Value }
+  }
+  catch {
+    Write-Warning "Failed to load weights: $($_.Exception.Message). Skipping weighted batching."
+    return
+  }
+}
+
+if ($weights.Count -eq 0) {
+  Write-Host "No weights available. Skipping weighted batching."
+  return
+}
+
+# Load all PackageInfo files
+$packageFiles = Get-ChildItem -Path $PackageInfoFolder -Filter "*.json" -Recurse
+$packages = @($packageFiles | ForEach-Object {
+  $json = Get-Content $_.FullName | ConvertFrom-Json
+  [PSCustomObject]@{
+    FilePath = $_.FullName
+    FileName = $_.Name
+    Json     = $json
+  }
+})
+
+if ($packages.Count -eq 0) {
+  Write-Host "No PackageInfo files found. Skipping weighted batching."
+  return
+}
+
+# Separate direct and indirect packages
+$directPackages = @($packages | Where-Object { $_.Json.IncludedForValidation -eq $false })
+$indirectPackages = @($packages | Where-Object { $_.Json.IncludedForValidation -eq $true })
+
+$totalPkgs = $directPackages.Count + $indirectPackages.Count
+$matchedWeights = @($packages | Where-Object { $weights.ContainsKey($_.Json.ArtifactName) }).Count
+Write-Host "Packages: $($directPackages.Count) direct, $($indirectPackages.Count) indirect ($matchedWeights/$totalPkgs have weights)"
+
+function Apply-LPTBatching {
+  param(
+    [object[]]$Packages,
+    [hashtable]$Weights,
+    [int]$TargetSeconds,
+    [int]$DefaultWeight,
+    [string]$Label
+  )
+
+  if ($Packages.Count -le 1) {
+    Write-Host "  $Label`: Only $($Packages.Count) package(s), no batching needed."
+    return
+  }
+
+  # Build weighted items
+  $items = @(foreach ($pkg in $Packages) {
+    $name = $pkg.Json.ArtifactName
+    $weight = if ($Weights.ContainsKey($name)) { [int]$Weights[$name] } else { $DefaultWeight }
+    [PSCustomObject]@{ Package = $pkg; Weight = $weight; Name = $name }
+  })
+
+  # Sort by weight descending (LPT: largest first)
+  $items = @($items | Sort-Object Weight -Descending)
+
+  # Calculate number of buckets
+  [int]$totalWeight = 0
+  foreach ($i in $items) { $totalWeight += $i.Weight }
+  $numBuckets = [math]::Max(1, [math]::Ceiling($totalWeight / $TargetSeconds))
+
+  # Don't create more buckets than packages
+  $numBuckets = [math]::Min($numBuckets, $Packages.Count)
+
+  Write-Host "  $Label`: $($Packages.Count) packages, total weight ${totalWeight}s, target ${TargetSeconds}s -> $numBuckets buckets"
+
+  # Create buckets
+  $buckets = @()
+  for ($b = 0; $b -lt $numBuckets; $b++) {
+    $buckets += [PSCustomObject]@{
+      Items       = [System.Collections.ArrayList]::new()
+      TotalWeight = [int]0
+    }
+  }
+
+  # Greedy LPT: assign each item to the lightest bucket
+  foreach ($item in $items) {
+    $lightest = $buckets | Sort-Object TotalWeight | Select-Object -First 1
+    [void]$lightest.Items.Add($item)
+    $lightest.TotalWeight += $item.Weight
+  }
+
+  # Log bucket distribution with collapsible groups
+  $bucketIdx = 1
+  foreach ($bucket in $buckets) {
+    $summary = "Bucket ${bucketIdx}: $($bucket.Items.Count) pkgs, weight $($bucket.TotalWeight)s ($([math]::Round($bucket.TotalWeight / 60, 1))m)"
+    Write-Host "##[group]$summary"
+    foreach ($item in ($bucket.Items | Sort-Object Name)) {
+      Write-Host "      $($item.Name) ($($item.Weight)s)"
+    }
+    Write-Host "##[endgroup]"
+    $bucketIdx++
+  }
+
+  # Consolidate: for each bucket, keep one representative file with all names
+  foreach ($bucket in $buckets) {
+    $batchItems = @($bucket.Items)
+    if ($batchItems.Count -eq 0) { continue }
+
+    # Use the first (heaviest) package as the representative
+    $representative = $batchItems[0].Package
+    $allNames = ($batchItems | ForEach-Object { $_.Name }) -join ","
+
+    # Update the representative's ArtifactName to contain all package names
+    $representative.Json.ArtifactName = $allNames
+    $representative.Json | ConvertTo-Json -Depth 100 | Set-Content $representative.FilePath -Encoding utf8
+
+    # Delete the other package files in this batch
+    for ($i = 1; $i -lt $batchItems.Count; $i++) {
+      Remove-Item $batchItems[$i].Package.FilePath -Force
+    }
+  }
+
+  Write-Host "  $Label`: Consolidated $($Packages.Count) files into $numBuckets batch files."
+}
+
+# Apply LPT batching to direct and indirect packages separately
+if ($directPackages.Count -gt 0) {
+  Apply-LPTBatching -Packages $directPackages -Weights $weights `
+    -TargetSeconds $TargetSeconds -DefaultWeight $DefaultWeight -Label "Direct"
+}
+
+if ($indirectPackages.Count -gt 0) {
+  Apply-LPTBatching -Packages $indirectPackages -Weights $weights `
+    -TargetSeconds $IndirectTargetSeconds -DefaultWeight $DefaultWeight -Label "Indirect"
+}
+
+# Verify
+$remaining = @(Get-ChildItem -Path $PackageInfoFolder -Filter "*.json" -Recurse).Count
+Write-Host "Weighted batching complete. $remaining consolidated PackageInfo files remain."