diff --git a/eng/pipelines/templates/jobs/batched-analyze.yml b/eng/pipelines/templates/jobs/batched-analyze.yml new file mode 100644 index 000000000000..5f3f83dff5b2 --- /dev/null +++ b/eng/pipelines/templates/jobs/batched-analyze.yml @@ -0,0 +1,52 @@ +parameters: + - name: Artifacts + type: object + default: [] + - name: ServiceDirectory + type: string + default: not-specified + - name: SDKType + type: string + default: all + - name: BuildSnippets + type: boolean + default: true + - name: ArtifactName + type: string + default: BuildPackagesArtifact + - name: DependsOn + type: string + default: '' + - name: CloudConfig + type: object + default: {} + - name: Matrix + type: string + - name: UsePlatformContainer + type: boolean + default: false + - name: OSName + type: string + +jobs: + - job: + displayName: "Analyze" + dependsOn: ${{ parameters.DependsOn }} + condition: and(succeededOrFailed(), ne(${{ parameters.Matrix }}, '{}')) + + strategy: + matrix: $[ ${{ parameters.Matrix }} ] + + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + + steps: + - template: /eng/pipelines/templates/steps/analyze.yml + parameters: + Artifacts: ${{ parameters.Artifacts }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + SDKType: ${{ parameters.SDKType }} + BuildSnippets: ${{ parameters.BuildSnippets }} + ArtifactName: ${{ parameters.ArtifactName }} diff --git a/eng/pipelines/templates/jobs/batched-build-analyze.yml b/eng/pipelines/templates/jobs/batched-build.yml similarity index 60% rename from eng/pipelines/templates/jobs/batched-build-analyze.yml rename to eng/pipelines/templates/jobs/batched-build.yml index a0df8ed6aa71..f34cd6364215 100644 --- a/eng/pipelines/templates/jobs/batched-build-analyze.yml +++ b/eng/pipelines/templates/jobs/batched-build.yml @@ -41,8 +41,6 @@ jobs: value: ${{ parameters.SDKType }} pool: - # 1es pipeline templates converts `image` to demands: ImageOverride under the hood - # which is incompatible with image selection in the default non-1es hosted pools ${{ if eq(parameters.OSName, 'macOS') }}: vmImage: $(OSVmImage) ${{ else }}: @@ -50,7 +48,6 @@ jobs: name: $(Pool) os: ${{ parameters.OSName }} ${{ if eq(parameters.UsePlatformContainer, 'true') }}: - # Add a default so the job doesn't fail when the matrix is empty container: $[ variables['Container'] ] steps: @@ -60,26 +57,3 @@ jobs: TestPipeline: ${{ parameters.TestPipeline }} ServiceDirectory: ${{ parameters.ServiceDirectory }} SDKType: ${{ parameters.SDKType }} - - - job: - displayName: "Analyze" - dependsOn: ${{ parameters.DependsOn }} - condition: and(succeededOrFailed(), ne(${{ parameters.Matrix }}, '{}')) - - # allow duplication of the job by matrix - strategy: - matrix: $[ ${{ parameters.Matrix }} ] - - # but immediately ignore the matrix for the job name - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - - steps: - - template: /eng/pipelines/templates/steps/analyze.yml - parameters: - Artifacts: ${{ parameters.Artifacts }} - ServiceDirectory: ${{ parameters.ServiceDirectory }} - SDKType: ${{ parameters.SDKType }} - BuildSnippets: ${{ parameters.BuildSnippets }} diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index c059f3307b3c..2f9eaebe0f71 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -45,10 +45,11 @@ parameters: jobs: - ${{ if eq(parameters.ServiceDirectory, 'auto') }}: + # Build matrix - larger LOC batches (1.2M) for fewer jobs and lower per-job overhead. - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml parameters: GenerateJobName: generate_build_matrix - JobTemplatePath: /eng/pipelines/templates/jobs/batched-build-analyze.yml + JobTemplatePath: /eng/pipelines/templates/jobs/batched-build.yml Pools: # eng/pipelines/templates/stages/build-matrix.json only contains windows OS currently so only pass the Windows pool to avoid the other OS's being marked skipped in DevOps - name: Windows filter: .*Windows.*Pool$ @@ -63,7 +64,7 @@ jobs: - "!SessionRecords" EnablePRGeneration: true PRMatrixSetting: ProjectNames - PRJobBatchSize: 5 + PRJobBatchSize: 1 AdditionalParameters: Artifacts: ${{ parameters.Artifacts }} TestPipeline: ${{ parameters.TestPipeline }} @@ -77,6 +78,46 @@ jobs: PublishingArtifactName: BuildPackagesArtifact ForceDirect: true ExcludePaths: ${{ parameters.ExcludePaths }} + EnableLocWeighting: true + LocTarget: 1200000 + PublishArtifact: true + + # Analyze matrix - smaller LOC batches (300k) for more parallelism so analyze wall-clock matches build. + - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml + parameters: + GenerateJobName: generate_analyze_matrix + JobTemplatePath: /eng/pipelines/templates/jobs/batched-analyze.yml + Pools: # build-matrix.json only contains Windows entries; batched-analyze.yml overrides pool to Linux internally. + - name: Windows + filter: .*Windows.*Pool$ + os: windows + MatrixConfigs: + - Name: NET_ci_analyze_base + Path: eng/pipelines/templates/stages/build-matrix.json + Selection: sparse + GenerateVMJobs: true + SparseCheckoutPaths: + - "/*" + - "!SessionRecords" + EnablePRGeneration: true + PRMatrixSetting: ProjectNames + PRJobBatchSize: 1 + AdditionalParameters: + Artifacts: ${{ parameters.Artifacts }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + SDKType: ${{ parameters.SDKType }} + BuildSnippets: ${{ parameters.BuildSnippets }} + ArtifactName: AnalyzePackageInfoArtifact + PreGenerationSteps: + - template: /eng/pipelines/templates/steps/pr-matrix-presteps.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + PublishingArtifactName: AnalyzePackageInfoArtifact + ForceDirect: true + ExcludePaths: ${{ parameters.ExcludePaths }} + EnableLocWeighting: true + LocTarget: 300000 + PublishArtifact: true - ${{ else }}: - job: Build diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index 7e4f438bab16..75647b1a9f16 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -11,6 +11,9 @@ parameters: - name: BuildSnippets type: boolean default: true + - name: ArtifactName + type: string + default: BuildPackagesArtifact steps: - ${{ if not(contains(variables['Build.DefinitionName'], '-pr - ')) }}: @@ -27,7 +30,7 @@ steps: - ${{ if eq(parameters.ServiceDirectory, 'auto') }}: - task: DownloadPipelineArtifact@2 inputs: - artifact: BuildPackagesArtifact + artifact: ${{ parameters.ArtifactName }} path: $(Build.ArtifactStagingDirectory)/PackageInfo - ${{ else }}: - template: /eng/common/pipelines/templates/steps/save-package-properties.yml diff --git a/eng/pipelines/templates/steps/pr-matrix-presteps.yml b/eng/pipelines/templates/steps/pr-matrix-presteps.yml index bd732d8bd2bd..ae42bc505b8d 100644 --- a/eng/pipelines/templates/steps/pr-matrix-presteps.yml +++ b/eng/pipelines/templates/steps/pr-matrix-presteps.yml @@ -9,6 +9,15 @@ parameters: - name: ExcludePaths type: object default: [] + - name: EnableLocWeighting + type: boolean + default: false + - name: LocTarget + type: number + default: 474000 + - name: PublishArtifact + type: boolean + default: true steps: - template: /eng/pipelines/templates/steps/install-dotnet.yml @@ -28,10 +37,11 @@ steps: | Copy-Item -Destination "$(Build.ArtifactStagingDirectory)/PackageInfo/$($_.Name)" -Force displayName: Copy ArtifactInfo from Publishing Directory - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - ArtifactName: '${{ parameters.PublishingArtifactName }}' - ArtifactPath: '$(Build.ArtifactStagingDirectory)/PackageInfoPublishing' + - ${{ if eq(parameters.PublishArtifact, true) }}: + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactName: '${{ parameters.PublishingArtifactName }}' + ArtifactPath: '$(Build.ArtifactStagingDirectory)/PackageInfoPublishing' - pwsh: | $packageInfos = Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/PackageInfo" -Filter "*.json" -Recurse ` @@ -62,3 +72,19 @@ steps: displayName: Only target direct packages for build and test condition: and(succeededOrFailed(), eq(${{ parameters.ForceDirect }}, true)) + - ${{ if eq(parameters.EnableLocWeighting, true) }}: + - pwsh: | + eng/scripts/Get-PackageLocWeights.ps1 ` + -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" ` + -RepoRoot "$(Build.SourcesDirectory)" ` + -OutputFile "$(Build.ArtifactStagingDirectory)/loc-weights.json" + displayName: Count LOC per package for build/analyze weighting + condition: and(succeededOrFailed(), eq(variables['Build.Reason'], 'PullRequest')) + + - pwsh: | + eng/scripts/Apply-WeightedBatching.ps1 ` + -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" ` + -WeightsFile "$(Build.ArtifactStagingDirectory)/loc-weights.json" ` + -Target ${{ parameters.LocTarget }} + displayName: Apply LOC-weighted batching (target ${{ parameters.LocTarget }}) + condition: and(succeededOrFailed(), eq(variables['Build.Reason'], 'PullRequest')) \ No newline at end of file diff --git a/eng/scripts/Apply-WeightedBatching.ps1 b/eng/scripts/Apply-WeightedBatching.ps1 new file mode 100644 index 000000000000..31153d79bb3b --- /dev/null +++ b/eng/scripts/Apply-WeightedBatching.ps1 @@ -0,0 +1,189 @@ +<# +.SYNOPSIS +Applies weighted LPT bin-packing to PackageInfo files for balanced CI job batching. + +.DESCRIPTION +Reads PackageInfo JSON files and a weights file, performs LPT (Longest Processing Time) +bin-packing to create balanced batches, then consolidates the PackageInfo files so that each +batch becomes a single representative file. The ArtifactName of each consolidated file contains +the comma-separated names of all packages in that batch, which flows through to the +ProjectNames parameter in the matrix generation. + +Used by the Build and Analyze pre-steps with LOC-derived weights, but the script itself is +weight-source-agnostic — any numeric weight file (package name -> integer) will work. + +After this script runs, set PRJobBatchSize to 1 so each consolidated file becomes its own job. + +.PARAMETER PackageInfoFolder +Path to the folder containing PackageInfo JSON files. + +.PARAMETER WeightsFile +Path to the JSON weights file (artifact name -> weight, e.g. LOC count). The key must match +the `ArtifactName` field of the corresponding PackageInfo JSON. + +.PARAMETER Target +Target average weight per bucket for direct packages. Default is 1800. Used to derive +the bucket count via ceil(totalWeight / Target); individual buckets may exceed this +after greedy LPT packing — it's a goal, not a hard cap. + +.PARAMETER IndirectTarget +Target average weight per bucket for indirect packages. Defaults to Target if not specified. +Indirect packages only run on Linux, so they can use a higher target than direct packages which run across all platforms. + +.PARAMETER DefaultWeight +Weight assigned to packages not found in the weights file. Default is 1. +#> + +[CmdletBinding()] +param ( + [Parameter(Mandatory = $true)][string]$PackageInfoFolder, + [Parameter(Mandatory = $true)][string]$WeightsFile, + [Parameter()][int]$Target = 1800, + [Parameter()][int]$IndirectTarget = 0, + [Parameter()][int]$DefaultWeight = 1 +) + +Set-StrictMode -Version 4 + +if ($IndirectTarget -le 0) { $IndirectTarget = $Target } + +# Load weights +$weights = @{} +if (Test-Path $WeightsFile) { + try { + $content = Get-Content $WeightsFile -Raw | ConvertFrom-Json + $content.PSObject.Properties | ForEach-Object { $weights[$_.Name] = [int]$_.Value } + } + catch { + Write-Warning "Failed to load weights: $($_.Exception.Message). Skipping weighted batching." + return + } +} + +if ($weights.Count -eq 0) { + Write-Host "No weights available. Skipping weighted batching." + return +} + +# Load all PackageInfo files +$packageFiles = Get-ChildItem -Path $PackageInfoFolder -Filter "*.json" -Recurse +$packages = @($packageFiles | ForEach-Object { + $json = Get-Content $_.FullName | ConvertFrom-Json + [PSCustomObject]@{ + FilePath = $_.FullName + FileName = $_.Name + Json = $json + } +}) + +if ($packages.Count -eq 0) { + Write-Host "No PackageInfo files found. Skipping weighted batching." + return +} + +# Separate direct and indirect packages +$directPackages = @($packages | Where-Object { $_.Json.IncludedForValidation -eq $false }) +$indirectPackages = @($packages | Where-Object { $_.Json.IncludedForValidation -eq $true }) + +$totalPkgs = $directPackages.Count + $indirectPackages.Count +$matchedWeights = @($packages | Where-Object { $weights.ContainsKey($_.Json.ArtifactName) }).Count +Write-Host "Packages: $($directPackages.Count) direct, $($indirectPackages.Count) indirect ($matchedWeights/$totalPkgs have weights)" + +function Apply-LPTBatching { + param( + [object[]]$Packages, + [hashtable]$Weights, + [int]$Target, + [int]$DefaultWeight, + [string]$Label + ) + + if ($Packages.Count -le 1) { + Write-Host " $Label`: Only $($Packages.Count) package(s), no batching needed." + return + } + + # Build weighted items + $items = @(foreach ($pkg in $Packages) { + $name = $pkg.Json.ArtifactName + $weight = if ($Weights.ContainsKey($name)) { [int]$Weights[$name] } else { $DefaultWeight } + [PSCustomObject]@{ Package = $pkg; Weight = $weight; Name = $name } + }) + + # Sort by weight descending (LPT: largest first) + $items = @($items | Sort-Object Weight -Descending) + + # Calculate number of buckets + [int]$totalWeight = 0 + foreach ($i in $items) { $totalWeight += $i.Weight } + $numBuckets = [math]::Max(1, [math]::Ceiling($totalWeight / $Target)) + + # Don't create more buckets than packages + $numBuckets = [math]::Min($numBuckets, $Packages.Count) + + Write-Host " $Label`: $($Packages.Count) packages, total weight ${totalWeight}, target ${Target} -> $numBuckets buckets" + + # Create buckets + $buckets = @() + for ($b = 0; $b -lt $numBuckets; $b++) { + $buckets += [PSCustomObject]@{ + Items = [System.Collections.ArrayList]::new() + TotalWeight = [int]0 + } + } + + # Greedy LPT: assign each item to the lightest bucket + foreach ($item in $items) { + $lightest = $buckets | Sort-Object TotalWeight | Select-Object -First 1 + [void]$lightest.Items.Add($item) + $lightest.TotalWeight += $item.Weight + } + + # Log bucket distribution with collapsible groups + $bucketIdx = 1 + foreach ($bucket in $buckets) { + $summary = "Bucket ${bucketIdx}: $($bucket.Items.Count) pkgs, weight $($bucket.TotalWeight)" + Write-Host "##[group]$summary" + foreach ($item in ($bucket.Items | Sort-Object Name)) { + Write-Host " $($item.Name) (weight $($item.Weight))" + } + Write-Host "##[endgroup]" + $bucketIdx++ + } + + # Consolidate: for each bucket, keep one representative file with all names + foreach ($bucket in $buckets) { + $batchItems = @($bucket.Items) + if ($batchItems.Count -eq 0) { continue } + + # Use the first (heaviest) package as the representative + $representative = $batchItems[0].Package + $allNames = ($batchItems | ForEach-Object { $_.Name }) -join "," + + # Update the representative's ArtifactName to contain all package names + $representative.Json.ArtifactName = $allNames + $representative.Json | ConvertTo-Json -Depth 100 | Set-Content $representative.FilePath -Encoding utf8 + + # Delete the other package files in this batch + for ($i = 1; $i -lt $batchItems.Count; $i++) { + Remove-Item $batchItems[$i].Package.FilePath -Force + } + } + + Write-Host " $Label`: Consolidated $($Packages.Count) files into $numBuckets batch files." +} + +# Apply LPT batching to direct and indirect packages separately +if ($directPackages.Count -gt 0) { + Apply-LPTBatching -Packages $directPackages -Weights $weights ` + -Target $Target -DefaultWeight $DefaultWeight -Label "Direct" +} + +if ($indirectPackages.Count -gt 0) { + Apply-LPTBatching -Packages $indirectPackages -Weights $weights ` + -Target $IndirectTarget -DefaultWeight $DefaultWeight -Label "Indirect" +} + +# Verify +$remaining = @(Get-ChildItem -Path $PackageInfoFolder -Filter "*.json" -Recurse).Count +Write-Host "Weighted batching complete. $remaining consolidated PackageInfo files remain." diff --git a/eng/scripts/Get-PackageLocWeights.ps1 b/eng/scripts/Get-PackageLocWeights.ps1 new file mode 100644 index 000000000000..db13fe821c92 --- /dev/null +++ b/eng/scripts/Get-PackageLocWeights.ps1 @@ -0,0 +1,68 @@ +<# +.SYNOPSIS +Counts lines of code per package and generates a weight file for build/analyze batching. + +.DESCRIPTION +For each PackageInfo JSON file, finds the corresponding src directory and counts +total lines of C# code. Outputs a JSON mapping of artifact name (the `ArtifactName` +field from PackageInfo) to LOC count. + +.PARAMETER PackageInfoFolder +Path to the folder containing PackageInfo JSON files. + +.PARAMETER RepoRoot +Root of the repository. + +.PARAMETER OutputFile +Path to write the LOC weights JSON file. +#> + +[CmdletBinding()] +param ( + [Parameter(Mandatory = $true)][string]$PackageInfoFolder, + [Parameter(Mandatory = $true)][string]$RepoRoot, + [Parameter(Mandatory = $true)][string]$OutputFile +) + +Set-StrictMode -Version 4 + +$weights = @{} +$packageFiles = Get-ChildItem -Path $PackageInfoFolder -Filter "*.json" -Recurse + +foreach ($file in $packageFiles) { + $json = Get-Content $file.FullName | ConvertFrom-Json + $name = $json.ArtifactName + $dirPath = $json.DirectoryPath + + if (-not $dirPath) { + $weights[$name] = 1 + continue + } + + if ([System.IO.Path]::IsPathRooted($dirPath)) { + $srcPath = Join-Path $dirPath "src" + } + else { + $srcPath = Join-Path $RepoRoot $dirPath "src" + } + if (-not (Test-Path $srcPath)) { + $weights[$name] = 1 + continue + } + + $loc = 0 + Get-ChildItem $srcPath -Filter "*.cs" -Recurse -ErrorAction SilentlyContinue | ForEach-Object { + $loc += @(Get-Content $_.FullName -ErrorAction SilentlyContinue).Count + } + + $weights[$name] = [math]::Max($loc, 1) +} + +Write-Host "Counted LOC for $($weights.Count) packages." +$topPkgs = $weights.GetEnumerator() | Sort-Object Value -Descending | Select-Object -First 5 +foreach ($p in $topPkgs) { + Write-Host " $($p.Key): $($p.Value) LOC" +} + +$weights | ConvertTo-Json -Depth 1 | Set-Content $OutputFile -Encoding utf8 +Write-Host "LOC weights written to $OutputFile"