diff --git a/.github/scripts/Invoke-RerunReviewTrigger.Tests.ps1 b/.github/scripts/Invoke-RerunReviewTrigger.Tests.ps1 index af0ef3343055..7d4554c45ddf 100644 --- a/.github/scripts/Invoke-RerunReviewTrigger.Tests.ps1 +++ b/.github/scripts/Invoke-RerunReviewTrigger.Tests.ps1 @@ -14,7 +14,7 @@ BeforeAll { $script:ReviewTriggerWindowHours = 24 $script:MaxReviewTriggersPerWindow = 3 - foreach ($functionName in @('Get-ReviewTriggerRateLimitStatus', 'ConvertTo-SafeLogValue', 'Get-MatchingCandidate', 'Normalize-PipelineRef', 'Get-PlatformFromLabels')) { + foreach ($functionName in @('Get-ReviewTriggerRateLimitStatus', 'ConvertTo-SafeLogValue', 'ConvertTo-TrimmedString', 'Test-GhApiPrNotFound', 'Get-MatchingCandidate', 'Normalize-PipelineRef', 'Get-PlatformFromLabels')) { $function = $ast.Find({ $args[0] -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $args[0].Name -eq $functionName @@ -73,6 +73,30 @@ Describe 'ConvertTo-SafeLogValue' { } } +Describe 'Test-GhApiPrNotFound' { + It 'recognizes stale PR responses' { + Test-GhApiPrNotFound 'gh: Not Found (HTTP 404)' | Should -BeTrue + Test-GhApiPrNotFound 'gh: Gone (HTTP 410)' | Should -BeTrue + } + + It 'does not hide credential, rate-limit, or transient failures' { + Test-GhApiPrNotFound 'gh: Bad credentials (HTTP 401)' | Should -BeFalse + Test-GhApiPrNotFound 'gh: API rate limit exceeded (HTTP 403)' | Should -BeFalse + Test-GhApiPrNotFound 'gh: Internal Server Error (HTTP 500)' | Should -BeFalse + Test-GhApiPrNotFound '' | Should -BeFalse + } +} + +Describe 'ConvertTo-TrimmedString' { + It 'returns empty string for null values' { + ConvertTo-TrimmedString $null | Should -Be '' + } + + It 'trims non-null values' { + ConvertTo-TrimmedString " ok`n" | Should -Be 'ok' + } +} + Describe 'Get-MatchingCandidate' { It 'matches only PRs in the deterministic candidate set' { $candidates = @( diff --git a/.github/scripts/Invoke-RerunReviewTrigger.ps1 b/.github/scripts/Invoke-RerunReviewTrigger.ps1 index 00313489a541..202bb125fcd0 100644 --- a/.github/scripts/Invoke-RerunReviewTrigger.ps1 +++ b/.github/scripts/Invoke-RerunReviewTrigger.ps1 @@ -68,6 +68,26 @@ function ConvertTo-SafeLogValue { return $safe } +function Test-GhApiPrNotFound { + param([string]$Output) + + if ([string]::IsNullOrWhiteSpace($Output)) { + return $false + } + + return $Output -match '(?i)\bHTTP\s+(404|410)\b' -or $Output -match '(?i)\b(Not Found|Gone)\b' +} + +function ConvertTo-TrimmedString { + param([AllowNull()][object]$Value) + + if ($null -eq $Value) { + return '' + } + + return ([string]$Value).Trim() +} + function Add-CommentReaction { param( [Parameter(Mandatory = $true)][Int64]$CommentId, @@ -270,6 +290,7 @@ if ($items.Count -eq 0) { exit 0 } $candidates = @(Get-CandidateItems -Path $env:RERUN_CANDIDATES_PATH) +$hadProcessingFailure = $false foreach ($item in $items) { $prNumber = 0 @@ -311,7 +332,33 @@ foreach ($item in $items) { } Write-Host "Processing PR #$prNumber decision=$decision reason=$(ConvertTo-SafeLogValue $reason)" - $pr = gh api "repos/$Owner/$Repo/pulls/$prNumber" | ConvertFrom-Json + $prStdErrFile = New-TemporaryFile + try { + $prOutput = @(& gh api "repos/$Owner/$Repo/pulls/$prNumber" 2> $prStdErrFile) + $prExitCode = $LASTEXITCODE + $prJson = ConvertTo-TrimmedString ($prOutput | Out-String) + $prStdErr = ConvertTo-TrimmedString (Get-Content -Raw -LiteralPath $prStdErrFile -ErrorAction SilentlyContinue) + } finally { + Remove-Item -LiteralPath $prStdErrFile -Force -ErrorAction SilentlyContinue + } + if ($prExitCode -ne 0) { + $prError = if ([string]::IsNullOrWhiteSpace($prStdErr)) { $prJson } else { $prStdErr } + if (Test-GhApiPrNotFound -Output $prError) { + $global:LASTEXITCODE = 0 + Write-Host " ⏭️ PR #$prNumber no longer exists; skipping stale decision" + continue + } + + throw "Failed to load PR #$prNumber via gh api: $(ConvertTo-SafeLogValue $prError)" + } + if ([string]::IsNullOrWhiteSpace($prJson)) { + throw "Failed to load PR #$prNumber via gh api: empty response." + } + try { + $pr = $prJson | ConvertFrom-Json + } catch { + throw "Failed to parse PR #$prNumber response from gh api: $(ConvertTo-SafeLogValue ([string]$_))" + } if ($pr.state -ne 'open') { Write-Host " ⏭️ PR #$prNumber is not open ($($pr.state)); skipping" continue @@ -399,6 +446,13 @@ foreach ($item in $items) { } catch { $target = if ($prNumber -gt 0) { "PR #$prNumber" } else { "agent decision" } Write-Host "::error::Failed to process $target`: $(ConvertTo-SafeLogValue ([string]$_))" + $hadProcessingFailure = $true continue } } + +if ($hadProcessingFailure) { + exit 1 +} + +exit 0 diff --git a/.github/scripts/Query-RerunReadyPRs.ps1 b/.github/scripts/Query-RerunReadyPRs.ps1 index 9aba7289a033..a02258be7e41 100644 --- a/.github/scripts/Query-RerunReadyPRs.ps1 +++ b/.github/scripts/Query-RerunReadyPRs.ps1 @@ -78,12 +78,16 @@ function Get-PlatformFromLabels { return 'android' } -$searchResult = gh pr list ` +$searchJson = gh pr list ` --repo "$Owner/$Repo" ` --state open ` --label $ReadyForRerunLabel ` --limit $MaxPRs ` - --json number,title,url,headRefOid,isDraft,labels | ConvertFrom-Json + --json number,title,url,headRefOid,isDraft,labels,author +if ($LASTEXITCODE -ne 0) { + throw "Failed to list open PRs labeled '$ReadyForRerunLabel' (gh pr list exited with code $LASTEXITCODE)." +} +$searchResult = $searchJson | ConvertFrom-Json $candidates = @() foreach ($pr in @($searchResult)) { @@ -101,7 +105,9 @@ foreach ($pr in @($searchResult)) { $latestRerun = Get-LatestRerunComment -Comments $activity $reviewOptionAuthors = @(Get-ReviewOptionAuthorLogins -Comments $activity) $reviewOptions = Get-LatestReviewCommandOptions -Comments $activity -AllowedAuthorLogins $reviewOptionAuthors - $contextMarkdown = New-RerunContextMarkdown -Comments $activity -Commits $commits -CurrentHeadSha $pr.headRefOid -CurrentLabels $labels + $rawAuthorLogin = if ($pr.author -and $pr.author.login) { [string]$pr.author.login } else { '' } + $authorLogin = Normalize-GitHubActorLogin $rawAuthorLogin + $contextMarkdown = New-RerunContextMarkdown -Comments $activity -Commits $commits -CurrentHeadSha $pr.headRefOid -PRAuthorLogin $authorLogin -CurrentLabels $labels $platform = if ($reviewOptions.Platform) { $reviewOptions.Platform } else { Get-PlatformFromLabels -Labels $labels } $pipelineRef = if ($reviewOptions.PipelineRef) { $reviewOptions.PipelineRef } else { 'main' } @@ -109,6 +115,7 @@ foreach ($pr in @($searchResult)) { prNumber = $number title = [string]$pr.title url = [string]$pr.url + authorLogin = $authorLogin isDraft = [bool]$pr.isDraft headSha = [string]$pr.headRefOid platform = $platform diff --git a/.github/scripts/Resolve-RerunEligibility.Tests.ps1 b/.github/scripts/Resolve-RerunEligibility.Tests.ps1 index c8fe02c34c59..340303cb8fcf 100644 --- a/.github/scripts/Resolve-RerunEligibility.Tests.ps1 +++ b/.github/scripts/Resolve-RerunEligibility.Tests.ps1 @@ -69,6 +69,12 @@ BeforeAll { } Describe 'Resolve-RerunEligibility' { + It 'normalizes app-style GitHub bot author logins' { + Normalize-GitHubActorLogin 'app/dependabot' | Should -Be 'dependabot[bot]' + Normalize-GitHubActorLogin ' dependabot[bot] ' | Should -Be 'dependabot[bot]' + Normalize-GitHubActorLogin '' | Should -Be '' + } + It 'parses review command branch and platform options for reruns' { $parsed = ConvertFrom-ReviewCommand '/review -b feature/regression-check -p ios' @@ -90,6 +96,11 @@ Describe 'Resolve-RerunEligibility' { Should -Be 'feature/regression-check' } + It 'normalizes app-style GitHub actor logins to bot logins' { + Normalize-GitHubActorLogin 'app/dependabot' | Should -Be 'dependabot[bot]' + Normalize-GitHubActorLogin 'dev-user' | Should -Be 'dev-user' + } + It 'finds latest normal review command while ignoring rerun and tests commands' { $comments = @( New-TestComment -Id 1 -Body '/review -b old/ref -p android' -CreatedAt '2026-05-31T09:00:00Z' @@ -193,17 +204,30 @@ Describe 'Resolve-RerunEligibility' { $result.Reason | Should -Be 'no-new-comments-or-commits' } - It 'accepts a non-command comment after the latest AI Summary' { + It 'accepts a non-command PR author comment after the latest AI Summary' { $comments = @( New-TestComment -Id 1 -Body (New-AISummaryBody) -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' New-TestComment -Id 2 -Body 'I pushed the requested update.' -CreatedAt '2026-05-31T09:45:00Z' New-TestComment -Id 10 -Body '/review rerun' -CreatedAt '2026-05-31T10:00:00Z' ) - $result = Resolve-RerunEligibility -Comments $comments -Commits @() -CurrentCommentId 10 -CurrentHeadSha 'abcdef123' + $result = Resolve-RerunEligibility -Comments $comments -Commits @() -CurrentCommentId 10 -CurrentHeadSha 'abcdef123' -PRAuthorLogin 'dev-user' $result.Eligible | Should -BeTrue - $result.Reason | Should -Be 'new-comment-after-ai-summary' + $result.Reason | Should -Be 'new-author-comment-after-ai-summary' + } + + It 'rejects a non-author maintainer comment after the latest AI Summary' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody) -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + New-TestComment -Id 2 -Body 'Could you please check the AI suggestions?' -CreatedAt '2026-05-31T09:45:00Z' -Login 'kubaflo' -Kind 'review' + New-TestComment -Id 10 -Body '/review rerun' -CreatedAt '2026-05-31T10:00:00Z' -Login 'kubaflo' + ) + + $result = Resolve-RerunEligibility -Comments $comments -Commits @() -CurrentCommentId 10 -CurrentHeadSha 'abcdef123' -PRAuthorLogin 'dev-user' + + $result.Eligible | Should -BeFalse + $result.Reason | Should -Be 'no-new-comments-or-commits' } It 'uses AI Summary creation time as the activity checkpoint when the summary was edited later' { @@ -213,10 +237,10 @@ Describe 'Resolve-RerunEligibility' { New-TestComment -Id 10 -Body '/review rerun' -CreatedAt '2026-05-31T10:00:00Z' ) - $result = Resolve-RerunEligibility -Comments $comments -Commits @() -CurrentCommentId 10 -CurrentHeadSha 'abcdef123' + $result = Resolve-RerunEligibility -Comments $comments -Commits @() -CurrentCommentId 10 -CurrentHeadSha 'abcdef123' -PRAuthorLogin 'dev-user' $result.Eligible | Should -BeTrue - $result.Reason | Should -Be 'new-comment-after-ai-summary' + $result.Reason | Should -Be 'new-author-comment-after-ai-summary' } It 'selects the newest AI Summary by creation time instead of edit time' { @@ -227,10 +251,10 @@ Describe 'Resolve-RerunEligibility' { New-TestComment -Id 10 -Body '/review rerun' -CreatedAt '2026-05-31T10:30:00Z' ) - $result = Resolve-RerunEligibility -Comments $comments -Commits @() -CurrentCommentId 10 -CurrentHeadSha '2222222abcdef' + $result = Resolve-RerunEligibility -Comments $comments -Commits @() -CurrentCommentId 10 -CurrentHeadSha '2222222abcdef' -PRAuthorLogin 'dev-user' $result.Eligible | Should -BeTrue - $result.Reason | Should -Be 'new-comment-after-ai-summary' + $result.Reason | Should -Be 'new-author-comment-after-ai-summary' } It 'ignores forged AI Summary comments from non-bots' { @@ -277,10 +301,10 @@ Describe 'Resolve-RerunEligibility' { New-TestComment -Id 4659999999 -Body '/review rerun' -CreatedAt '2026-06-09T09:00:00Z' ) - $result = Resolve-RerunEligibility -Comments $comments -Commits @() -CurrentCommentId 4659999999 -CurrentHeadSha '6e9af5bc8b5d0023400d653500951fb46df44170' + $result = Resolve-RerunEligibility -Comments $comments -Commits @() -CurrentCommentId 4659999999 -CurrentHeadSha '6e9af5bc8b5d0023400d653500951fb46df44170' -PRAuthorLogin 'dev-user' $result.Eligible | Should -BeTrue - $result.Reason | Should -Be 'new-comment-after-ai-summary' + $result.Reason | Should -Be 'new-author-comment-after-ai-summary' } It 'uses the first session marker from an AI Summary' { @@ -308,7 +332,7 @@ new $result.Reason | Should -Be 'no-new-comments-or-commits' } - It 'accepts a non-command comment after the previous rerun command' { + It 'accepts a non-command PR author comment after the previous rerun command' { $comments = @( New-TestComment -Id 1 -Body (New-AISummaryBody) -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' New-TestComment -Id 8 -Body '/review rerun' -CreatedAt '2026-05-31T09:45:00Z' @@ -316,10 +340,10 @@ new New-TestComment -Id 10 -Body '/review rerun' -CreatedAt '2026-05-31T10:00:00Z' ) - $result = Resolve-RerunEligibility -Comments $comments -Commits @() -CurrentCommentId 10 -CurrentHeadSha 'abcdef123' + $result = Resolve-RerunEligibility -Comments $comments -Commits @() -CurrentCommentId 10 -CurrentHeadSha 'abcdef123' -PRAuthorLogin 'dev-user' $result.Eligible | Should -BeTrue - $result.Reason | Should -Be 'new-comment-after-previous-rerun' + $result.Reason | Should -Be 'new-author-comment-after-previous-rerun' } It 'does not reuse old activity from before a previous rerun command' { @@ -343,10 +367,10 @@ new New-TestComment -Id 10 -Body '/review rerun' -CreatedAt '2026-05-31T10:00:00Z' ) - $result = Resolve-RerunEligibility -Comments $comments -Commits @() -CurrentCommentId 10 -CurrentHeadSha 'abcdef123' + $result = Resolve-RerunEligibility -Comments $comments -Commits @() -CurrentCommentId 10 -CurrentHeadSha 'abcdef123' -PRAuthorLogin 'dev-user' $result.Eligible | Should -BeTrue - $result.Reason | Should -Be 'new-comment-after-ai-summary' + $result.Reason | Should -Be 'new-author-comment-after-ai-summary' } It 'accepts a current head SHA that differs from the latest reviewed session' { @@ -418,21 +442,35 @@ new $comments = @( New-TestComment -Id 1 -Body (New-AISummaryBody) -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' New-TestComment -Id 2 -Body 'New author context.' -CreatedAt '2026-05-31T09:45:00Z' + New-TestComment -Id 11 -Body 'Reviewer reminder.' -CreatedAt '2026-05-31T09:46:00Z' -Login 'reviewer' New-TestComment -Id 3 -Body '/review rerun' -CreatedAt '2026-05-31T09:50:00Z' ) $commits = @( New-TestCommit -Sha 'fedcba9876543210' -Date '2026-05-31T09:48:00Z' ) - $context = New-RerunContextMarkdown -Comments $comments -Commits $commits -CurrentHeadSha 'fedcba9876543210' -CurrentLabels @('s/agent-review-in-progress') + $context = New-RerunContextMarkdown -Comments $comments -Commits $commits -CurrentHeadSha 'fedcba9876543210' -PRAuthorLogin 'dev-user' -CurrentLabels @('s/agent-review-in-progress') $context | Should -Match '# Rerun Context' - $context | Should -Match 'New non-command comments: 1' + $context | Should -Match 'New non-command author comments: 1' $context | Should -Match 'New commits: 1' $context | Should -Match '`s/agent-ready-for-rerun` present: false' $context | Should -Match '`s/agent-review-in-progress` present: true' $context | Should -Match 'New author context' + $context | Should -Not -Match 'Reviewer reminder' $context | Should -Match 'fedcba9' $context | Should -Not -Match '\| .*\/review rerun' } + + It 'renders normalized app-style bot authors in rerun context' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody) -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + New-TestComment -Id 3 -Body '/review rerun' -CreatedAt '2026-05-31T09:50:00Z' + ) + + $context = New-RerunContextMarkdown -Comments $comments -Commits @() -CurrentHeadSha 'abcdef123' -PRAuthorLogin 'app/dependabot' + + $context | Should -Match 'PR author: dependabot\[bot\]' + $context | Should -Match 'New non-command author comments: 0' + } } diff --git a/.github/scripts/Resolve-RerunEligibility.ps1 b/.github/scripts/Resolve-RerunEligibility.ps1 index 0b8ebcaaf48f..55351c68464b 100644 --- a/.github/scripts/Resolve-RerunEligibility.ps1 +++ b/.github/scripts/Resolve-RerunEligibility.ps1 @@ -6,8 +6,8 @@ .DESCRIPTION This script is intentionally deterministic: it never uses AI and never inspects untrusted text semantically. A rerun is eligible only when there is - new PR activity after the previous AI Summary or previous /review rerun: - a new non-command comment, or a new commit. + new PR-author activity after the previous AI Summary or previous /review rerun: + a new non-command PR-author comment, or a new commit. #> param( @@ -27,7 +27,7 @@ $ErrorActionPreference = 'Stop' $AISummaryMarker = '' $ReadyForRerunLabel = 's/agent-ready-for-rerun' $ReviewInProgressLabel = 's/agent-review-in-progress' -$ReadyForRerunLabelDescription = 'AI review has new PR activity and is ready for rerun' +$ReadyForRerunLabelDescription = 'AI review has a new PR-author comment or commit and is ready for rerun' $ReadyForRerunLabelColor = '5319E7' $AISummaryAuthorLogins = @( 'MauiBot' @@ -276,15 +276,42 @@ function Get-LatestReviewedSha { return $matches[0].Groups[1].Value.ToLowerInvariant() } +function Normalize-GitHubActorLogin { + param([string]$Login) + + if ([string]::IsNullOrWhiteSpace($Login)) { + return '' + } + + $trimmed = $Login.Trim() + if ($trimmed -match '^app/([^/\s]+)$') { + return "$($Matches[1])[bot]" + } + + return $trimmed +} + function Test-CommentIsEvidence { param( [Parameter(Mandatory = $true)]$Comment, - [Parameter(Mandatory = $true)][Int64]$CurrentCommentId + [Parameter(Mandatory = $true)][Int64]$CurrentCommentId, + [string]$PRAuthorLogin ) if ([Int64]$Comment.id -eq $CurrentCommentId) { return $false } + if ([string]::IsNullOrWhiteSpace($PRAuthorLogin)) { + return $false + } + if (-not $Comment.user -or [string]::IsNullOrWhiteSpace([string]$Comment.user.login)) { + return $false + } + $normalizedAuthorLogin = Normalize-GitHubActorLogin $PRAuthorLogin + $normalizedCommentLogin = Normalize-GitHubActorLogin ([string]$Comment.user.login) + if (-not $normalizedCommentLogin.Equals($normalizedAuthorLogin, [StringComparison]::OrdinalIgnoreCase)) { + return $false + } if (Test-RerunCommand $Comment.body) { return $false } @@ -302,11 +329,12 @@ function Test-HasEvidenceCommentAfter { param( [object[]]$Comments, [Parameter(Mandatory = $true)][datetimeoffset]$Checkpoint, - [Parameter(Mandatory = $true)][Int64]$CurrentCommentId + [Parameter(Mandatory = $true)][Int64]$CurrentCommentId, + [string]$PRAuthorLogin ) return [bool]@($Comments | Where-Object { - (Test-CommentIsEvidence -Comment $_ -CurrentCommentId $CurrentCommentId) -and + (Test-CommentIsEvidence -Comment $_ -CurrentCommentId $CurrentCommentId -PRAuthorLogin $PRAuthorLogin) -and (Get-ObjectDate $_ 'created_at') -gt $Checkpoint } | Select-Object -First 1) } @@ -401,12 +429,14 @@ function New-RerunContextMarkdown { [object[]]$Comments, [object[]]$Commits, [string]$CurrentHeadSha, + [string]$PRAuthorLogin, [object[]]$CurrentLabels = @() ) $latestSummary = Get-LatestAISummaryComment -Comments $Comments $latestRerun = Get-LatestRerunComment -Comments $Comments $checkpointRerun = if ($latestRerun) { Get-LatestRerunCommentBefore -Comments $Comments -CurrentCommentId ([Int64]$latestRerun.id) } else { $null } + $normalizedPRAuthorLogin = Normalize-GitHubActorLogin $PRAuthorLogin $readyLabelPresent = @($CurrentLabels | Where-Object { $_ -eq $ReadyForRerunLabel }).Count -gt 0 $inProgressLabelPresent = @($CurrentLabels | Where-Object { $_ -eq $ReviewInProgressLabel }).Count -gt 0 @@ -426,7 +456,7 @@ function New-RerunContextMarkdown { $evidenceComments = @() if ($checkpoint) { $evidenceComments = @($Comments | Where-Object { - (Test-CommentIsEvidence -Comment $_ -CurrentCommentId 0) -and + (Test-CommentIsEvidence -Comment $_ -CurrentCommentId 0 -PRAuthorLogin $normalizedPRAuthorLogin) -and (Get-ObjectDate $_ 'created_at') -gt $checkpoint } | Sort-Object @{ Expression = { Get-ObjectDate $_ 'created_at' }; Descending = $false }, @{ Expression = { [Int64]$_.id }; Descending = $false }) } @@ -465,6 +495,7 @@ function New-RerunContextMarkdown { } else { $lines.Add('- Activity checkpoint: none') } + $lines.Add("- PR author: $(if ([string]::IsNullOrWhiteSpace($normalizedPRAuthorLogin)) { 'unknown' } else { $normalizedPRAuthorLogin })") $lines.Add("- Latest reviewed SHA: $(if ($latestReviewedSha) { $latestReviewedSha } else { 'unknown' })") $lines.Add("- Current head SHA: $(if ($CurrentHeadSha) { $CurrentHeadSha } else { 'unknown' })") $lines.Add("- Current head differs from latest reviewed SHA: $($headDiffers.ToString().ToLowerInvariant())") @@ -473,7 +504,7 @@ function New-RerunContextMarkdown { $lines.Add('') $lines.Add('## New activity since checkpoint') $lines.Add('') - $lines.Add("- New non-command comments: $($evidenceComments.Count)") + $lines.Add("- New non-command author comments: $($evidenceComments.Count)") $lines.Add("- New commits: $($newCommits.Count)") $lines.Add('') @@ -519,6 +550,7 @@ function Resolve-RerunEligibility { [object[]]$Commits, [Parameter(Mandatory = $true)][Int64]$CurrentCommentId, [string]$CurrentHeadSha, + [string]$PRAuthorLogin, [object[]]$CurrentLabels = @() ) @@ -565,8 +597,9 @@ function Resolve-RerunEligibility { return [pscustomobject]@{ Eligible = $true; Reason = 'new-head-commit'; Label = $ReadyForRerunLabel } } - if (Test-HasEvidenceCommentAfter -Comments $Comments -Checkpoint $checkpoint -CurrentCommentId $CurrentCommentId) { - $reason = if ($checkpointReason -eq 'previous-rerun') { 'new-comment-after-previous-rerun' } else { 'new-comment-after-ai-summary' } + $normalizedPRAuthorLogin = Normalize-GitHubActorLogin $PRAuthorLogin + if (Test-HasEvidenceCommentAfter -Comments $Comments -Checkpoint $checkpoint -CurrentCommentId $CurrentCommentId -PRAuthorLogin $normalizedPRAuthorLogin) { + $reason = if ($checkpointReason -eq 'previous-rerun') { 'new-author-comment-after-previous-rerun' } else { 'new-author-comment-after-ai-summary' } return [pscustomobject]@{ Eligible = $true; Reason = $reason; Label = $ReadyForRerunLabel } } @@ -603,6 +636,7 @@ if ($ContextOutputPath) { -Comments $comments ` -Commits $commits ` -CurrentHeadSha $pr.head.sha ` + -PRAuthorLogin $pr.user.login ` -CurrentLabels $labels $contextDir = Split-Path -Parent $ContextOutputPath if ($contextDir) { @@ -627,6 +661,7 @@ $result = Resolve-RerunEligibility ` -Commits $commits ` -CurrentCommentId $CurrentCommentId ` -CurrentHeadSha $pr.head.sha ` + -PRAuthorLogin $pr.user.login ` -CurrentLabels $labels Write-Host "Rerun eligibility: $($result.Eligible) ($($result.Reason))" diff --git a/.github/workflows/rerun-review-scanner.lock.yml b/.github/workflows/rerun-review-scanner.lock.yml index c67521c099dc..aca5972bde6c 100644 --- a/.github/workflows/rerun-review-scanner.lock.yml +++ b/.github/workflows/rerun-review-scanner.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"2b9a2782d3cbe381a033db506d34d2bccce35ba8ef4668736e340068ccddbb4f","body_hash":"75dc74a551ad2ba0c8b1056bda890bce75dc6fa3ebf0536ad5a4eb46a61dcd5a","compiler_version":"v0.77.5","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"1202c20eb8b072cbb54e89173c40201cba38719b2d13e3dfd7517fc32303375d","body_hash":"bb8c799323c000bdf2eba2eb374a7843dc3066beae6dfc7337c6e0d54c884c10","compiler_version":"v0.77.5","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["AZDO_TRIGGER_CLIENT_ID","AZDO_TRIGGER_TENANT_ID","COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"34e114876b0b11c390a56381ad16ebd13914f8d5","version":"v4"},{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"3ea13c02d765410340d533515cb31a7eef2baaf0","version":"v0.77.5"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.58"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.22"},{"image":"ghcr.io/github/github-mcp-server:v1.1.0"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} # ___ _ _ # / _ \ | | (_) @@ -53,6 +53,42 @@ name: "Rerun Review Scanner" on: schedule: - cron: "0 * * * *" + # steps: # Steps injected into pre-activation job + # - name: Checkout repository scripts + # uses: actions/checkout@v4 + # with: + # persist-credentials: false + # - env: + # GH_TOKEN: ${{ github.token }} + # MAX_PRS: ${{ inputs.max_prs || '5' }} + # REPO_NAME: ${{ github.event.repository.name }} + # REPO_OWNER: ${{ github.repository_owner }} + # id: rerun_context + # name: Build rerun candidate context + # run: | + # $max = 5 + # if ($env:MAX_PRS -match '^\d+$') { + # $max = [Math]::Max(1, [Math]::Min(20, [int]$env:MAX_PRS)) + # } + # $output = "CustomAgentLogsTmp/RerunScanner/candidates.json" + # .github/scripts/Query-RerunReadyPRs.ps1 ` + # -Owner $env:REPO_OWNER ` + # -Repo $env:REPO_NAME ` + # -MaxPRs $max ` + # -OutputPath $output | Out-Null + # $json = Get-Content -Raw -LiteralPath $output + # $delimiter = "EOF_$([Guid]::NewGuid().ToString('N'))" + # "candidates<<$delimiter" >> $env:GITHUB_OUTPUT + # $json >> $env:GITHUB_OUTPUT + # $delimiter >> $env:GITHUB_OUTPUT + # shell: pwsh + # - name: Upload rerun candidate context + # uses: actions/upload-artifact@v7.0.1 + # with: + # if-no-files-found: error + # name: rerun-candidates + # path: CustomAgentLogsTmp/RerunScanner/candidates.json + # retention-days: 1 workflow_dispatch: inputs: aw_context: @@ -81,6 +117,8 @@ run-name: "Rerun Review Scanner" jobs: activation: + needs: pre_activation + if: needs.pre_activation.outputs.activated == 'true' runs-on: ubuntu-slim permissions: actions: read @@ -103,6 +141,8 @@ jobs: with: destination: ${{ runner.temp }}/gh-aw/actions job-name: ${{ github.job }} + trace-id: ${{ needs.pre_activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.pre_activation.outputs.setup-parent-span-id || needs.pre_activation.outputs.setup-span-id }} env: GH_AW_SETUP_WORKFLOW_NAME: "Rerun Review Scanner" GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/rerun-review-scanner.lock.yml@${{ github.ref }} @@ -196,25 +236,25 @@ jobs: GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - GH_AW_STEPS_RERUN_CONTEXT_OUTPUTS_CANDIDATES: ${{ steps.rerun_context.outputs.candidates }} + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_RERUN_CANDIDATES: ${{ needs.pre_activation.outputs.rerun_candidates }} # poutine:ignore untrusted_checkout_exec run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_c5f8a4ab1dfba5c0_EOF' + cat << 'GH_AW_PROMPT_cae32697d63d8db5_EOF' - GH_AW_PROMPT_c5f8a4ab1dfba5c0_EOF + GH_AW_PROMPT_cae32697d63d8db5_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_c5f8a4ab1dfba5c0_EOF' + cat << 'GH_AW_PROMPT_cae32697d63d8db5_EOF' Tools: missing_tool, missing_data, noop, trigger_rerun_review - GH_AW_PROMPT_c5f8a4ab1dfba5c0_EOF + GH_AW_PROMPT_cae32697d63d8db5_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" - cat << 'GH_AW_PROMPT_c5f8a4ab1dfba5c0_EOF' + cat << 'GH_AW_PROMPT_cae32697d63d8db5_EOF' The following GitHub context information is available for this workflow: {{#if github.actor}} @@ -243,19 +283,19 @@ jobs: {{/if}} - GH_AW_PROMPT_c5f8a4ab1dfba5c0_EOF + GH_AW_PROMPT_cae32697d63d8db5_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_c5f8a4ab1dfba5c0_EOF' + cat << 'GH_AW_PROMPT_cae32697d63d8db5_EOF' {{#runtime-import .github/workflows/rerun-review-scanner.md}} - GH_AW_PROMPT_c5f8a4ab1dfba5c0_EOF + GH_AW_PROMPT_cae32697d63d8db5_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_ENGINE_ID: "copilot" - GH_AW_STEPS_RERUN_CONTEXT_OUTPUTS_CANDIDATES: ${{ steps.rerun_context.outputs.candidates }} + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_RERUN_CANDIDATES: ${{ needs.pre_activation.outputs.rerun_candidates }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); @@ -275,7 +315,8 @@ jobs: GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` — run `safeoutputs --help` to see available tools' - GH_AW_STEPS_RERUN_CONTEXT_OUTPUTS_CANDIDATES: ${{ steps.rerun_context.outputs.candidates }} + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: ${{ needs.pre_activation.outputs.activated }} + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_RERUN_CANDIDATES: ${{ needs.pre_activation.outputs.rerun_candidates }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); @@ -296,7 +337,8 @@ jobs: GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST, - GH_AW_STEPS_RERUN_CONTEXT_OUTPUTS_CANDIDATES: process.env.GH_AW_STEPS_RERUN_CONTEXT_OUTPUTS_CANDIDATES + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED, + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_RERUN_CANDIDATES: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_RERUN_CANDIDATES } }); - name: Validate prompt placeholders @@ -393,23 +435,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - env: - GH_TOKEN: ${{ github.token }} - MAX_PRS: ${{ inputs.max_prs || '5' }} - REPO_NAME: ${{ github.event.repository.name }} - REPO_OWNER: ${{ github.repository_owner }} - id: rerun_context - name: Build rerun candidate context - run: "$max = 5\nif ($env:MAX_PRS -match '^\\d+$') {\n $max = [Math]::Max(1, [Math]::Min(20, [int]$env:MAX_PRS))\n}\n$output = \"CustomAgentLogsTmp/RerunScanner/candidates.json\"\n.github/scripts/Query-RerunReadyPRs.ps1 `\n -Owner $env:REPO_OWNER `\n -Repo $env:REPO_NAME `\n -MaxPRs $max `\n -OutputPath $output | Out-Null\n$json = Get-Content -Raw -LiteralPath $output\n$delimiter = \"EOF_$([Guid]::NewGuid().ToString('N'))\"\n\"candidates<<$delimiter\" >> $env:GITHUB_OUTPUT\n$json >> $env:GITHUB_OUTPUT\n$delimiter >> $env:GITHUB_OUTPUT\n" - shell: pwsh - - name: Upload rerun candidate context - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: error - name: rerun-candidates - path: CustomAgentLogsTmp/RerunScanner/candidates.json - retention-days: 1 - - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} @@ -480,9 +505,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_b87c598b007edcc9_EOF' + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_4cac70a2707e027b_EOF' {"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{},"trigger-rerun-review":{"description":"Apply a validated rerun scanner decision. Use once per candidate PR with decision 'trigger' or 'skip'.","inputs":{"decision":{"default":null,"description":"Whether to trigger or skip the rerun","options":["trigger","skip"],"required":true,"type":"choice"},"expected_head_sha":{"default":null,"description":"Current PR head SHA observed by the scanner","required":true,"type":"string"},"pipeline_ref":{"default":null,"description":"AzDO pipeline branch/ref to use for the rerun","required":false,"type":"string"},"platform":{"default":null,"description":"Optional target platform; leave empty to infer from labels","required":false,"type":"string"},"pr_number":{"default":null,"description":"Pull request number to process","required":true,"type":"string"},"reason":{"default":null,"description":"Short deterministic-safe reason for the decision","required":true,"type":"string"},"rerun_comment_id":{"default":null,"description":"Issue comment ID for the /review rerun command","required":true,"type":"string"}},"output":"Rerun scanner decision processed."}} - GH_AW_SAFE_OUTPUTS_CONFIG_b87c598b007edcc9_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_4cac70a2707e027b_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -702,7 +727,7 @@ jobs: mkdir -p /home/runner/.copilot GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_953a0c607e8bafff_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_c266c93f4f8f2521_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { @@ -743,7 +768,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_953a0c607e8bafff_EOF + GH_AW_MCP_CONFIG_c266c93f4f8f2521_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true @@ -1350,6 +1375,77 @@ jobs: } } + pre_activation: + runs-on: ubuntu-slim + outputs: + activated: ${{ steps.check_membership.outputs.is_team_member == 'true' }} + matched_command: '' + rerun_candidates: ${{ steps.rerun_context.outputs.candidates }} + rerun_context_result: ${{ steps.rerun_context.outcome }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + steps: + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@3ea13c02d765410340d533515cb31a7eef2baaf0 # v0.77.5 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Rerun Review Scanner" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/rerun-review-scanner.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.55" + GH_AW_INFO_AWF_VERSION: "v0.25.58" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Check team membership for workflow + id: check_membership + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_REQUIRED_ROLES: "admin,maintainer,write" + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_membership.cjs'); + await main(); + - name: Checkout repository scripts + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + - name: Build rerun candidate context + id: rerun_context + run: | + $max = 5 + if ($env:MAX_PRS -match '^\d+$') { + $max = [Math]::Max(1, [Math]::Min(20, [int]$env:MAX_PRS)) + } + $output = "CustomAgentLogsTmp/RerunScanner/candidates.json" + .github/scripts/Query-RerunReadyPRs.ps1 ` + -Owner $env:REPO_OWNER ` + -Repo $env:REPO_NAME ` + -MaxPRs $max ` + -OutputPath $output | Out-Null + $json = Get-Content -Raw -LiteralPath $output + $delimiter = "EOF_$([Guid]::NewGuid().ToString('N'))" + "candidates<<$delimiter" >> $env:GITHUB_OUTPUT + $json >> $env:GITHUB_OUTPUT + $delimiter >> $env:GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + MAX_PRS: ${{ inputs.max_prs || '5' }} + REPO_NAME: ${{ github.event.repository.name }} + REPO_OWNER: ${{ github.repository_owner }} + shell: pwsh + - name: Upload rerun candidate context + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: error + name: rerun-candidates + path: CustomAgentLogsTmp/RerunScanner/candidates.json + retention-days: 1 + safe_outputs: needs: - activation diff --git a/.github/workflows/rerun-review-scanner.md b/.github/workflows/rerun-review-scanner.md index 72ebd2039b46..8f8cdb0da13f 100644 --- a/.github/workflows/rerun-review-scanner.md +++ b/.github/workflows/rerun-review-scanner.md @@ -14,12 +14,53 @@ on: required: false type: number default: 5 + steps: + - name: Checkout repository scripts + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Build rerun candidate context + id: rerun_context + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + MAX_PRS: ${{ inputs.max_prs || '5' }} + REPO_OWNER: ${{ github.repository_owner }} + REPO_NAME: ${{ github.event.repository.name }} + run: | + $max = 5 + if ($env:MAX_PRS -match '^\d+$') { + $max = [Math]::Max(1, [Math]::Min(20, [int]$env:MAX_PRS)) + } + $output = "CustomAgentLogsTmp/RerunScanner/candidates.json" + .github/scripts/Query-RerunReadyPRs.ps1 ` + -Owner $env:REPO_OWNER ` + -Repo $env:REPO_NAME ` + -MaxPRs $max ` + -OutputPath $output | Out-Null + $json = Get-Content -Raw -LiteralPath $output + $delimiter = "EOF_$([Guid]::NewGuid().ToString('N'))" + "candidates<<$delimiter" >> $env:GITHUB_OUTPUT + $json >> $env:GITHUB_OUTPUT + $delimiter >> $env:GITHUB_OUTPUT + - name: Upload rerun candidate context + uses: actions/upload-artifact@v7.0.1 + with: + name: rerun-candidates + path: CustomAgentLogsTmp/RerunScanner/candidates.json + if-no-files-found: error + retention-days: 1 permissions: contents: read issues: read pull-requests: read +jobs: + pre-activation: + outputs: + rerun_candidates: ${{ steps.rerun_context.outputs.candidates }} + concurrency: # Serialize scheduled and manual scanner runs so each queued PR is evaluated # against the latest label/head/lock state before any safe-output job can trigger. @@ -103,38 +144,6 @@ safe-outputs: } .github/scripts/Invoke-RerunReviewTrigger.ps1 @scriptArgs -steps: - - name: Build rerun candidate context - id: rerun_context - shell: pwsh - env: - GH_TOKEN: ${{ github.token }} - MAX_PRS: ${{ inputs.max_prs || '5' }} - REPO_OWNER: ${{ github.repository_owner }} - REPO_NAME: ${{ github.event.repository.name }} - run: | - $max = 5 - if ($env:MAX_PRS -match '^\d+$') { - $max = [Math]::Max(1, [Math]::Min(20, [int]$env:MAX_PRS)) - } - $output = "CustomAgentLogsTmp/RerunScanner/candidates.json" - .github/scripts/Query-RerunReadyPRs.ps1 ` - -Owner $env:REPO_OWNER ` - -Repo $env:REPO_NAME ` - -MaxPRs $max ` - -OutputPath $output | Out-Null - $json = Get-Content -Raw -LiteralPath $output - $delimiter = "EOF_$([Guid]::NewGuid().ToString('N'))" - "candidates<<$delimiter" >> $env:GITHUB_OUTPUT - $json >> $env:GITHUB_OUTPUT - $delimiter >> $env:GITHUB_OUTPUT - - name: Upload rerun candidate context - uses: actions/upload-artifact@v7.0.1 - with: - name: rerun-candidates - path: CustomAgentLogsTmp/RerunScanner/candidates.json - if-no-files-found: error - retention-days: 1 --- # Rerun Review Scanner @@ -175,7 +184,7 @@ using a global concurrency group that could cancel unrelated maintainer The deterministic scanner found these candidates: ```json -${{ steps.rerun_context.outputs.candidates }} +${{ needs.pre_activation.outputs.rerun_candidates }} ``` For each candidate in `candidates`: diff --git a/.github/workflows/skill-validation.yml b/.github/workflows/skill-validation.yml index 1aa2241dd4cb..45f29701e87d 100644 --- a/.github/workflows/skill-validation.yml +++ b/.github/workflows/skill-validation.yml @@ -677,6 +677,7 @@ jobs: HAS_ENTRIES: ${{ needs.discover-eval.outputs.has_entries }} DISCOVER_RESULT: ${{ needs.discover-eval.result }} IS_CONTRIBUTOR: ${{ needs.pr-gate.outputs.is_contributor || 'true' }} + HEAD_SHA: ${{ needs.pr-gate.outputs.head_sha || needs.slash-gate.outputs.head_sha }} with: script: | const fs = require('fs'); @@ -685,6 +686,39 @@ jobs: const prNumber = parseInt(process.env.PR_NUMBER, 10); const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; const marker = ''; + // Metadata fetch is decorative; never let a transient failure (rate-limit/5xx) sink the whole results comment. + let prData = {}; + try { + const r = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + prData = r.data; + } catch (e) { + console.log('pulls.get failed, using fallbacks:', e.message); + } + // Use the gated SHA the validation actually ran against — the live PR head may have moved during a slow /evaluate-skills run. + const headSha = process.env.HEAD_SHA || prData.head?.sha || ''; + const headSha7 = headSha ? headSha.substring(0, 7) : 'unknown'; + const commitUrl = headSha ? `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/commit/${headSha}` : runUrl; + const prTitle = (prData.title || '').replace(/&/g, '&').replace(//g, '>'); + const prAuthor = prData.user?.login || ''; + const timestamp = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'); + + function badge(label, message, color) { + const safeLabel = encodeURIComponent(label).replace(/-/g, '--'); + const safeMessage = encodeURIComponent(message).replace(/-/g, '--'); + const safeAlt = `${label} ${message}`.replace(/&/g, '&').replace(//g, '>'); + return ` ${safeAlt}`; + } + + function colorFor(status) { + if (status === 'Passed') return '1a7f37'; + if (status === 'Failed') return 'd1242f'; + if (status === 'Skipped') return '6e7781'; + return 'bf8700'; + } const staticResult = process.env.STATIC_RESULT; const evalResult = process.env.EVAL_RESULT; @@ -693,7 +727,7 @@ jobs: const isContributor = process.env.IS_CONTRIBUTOR === 'true'; const evalRan = discoverResult === 'success'; - const lines = [marker, '## 🔍 Skill Validation Results', '']; + const lines = []; // ── Static check section ────────────────────────────── let staticOutput = ''; @@ -989,7 +1023,68 @@ jobs: // ── Pipeline link (styled like dotnet/skills) ───────── lines.push(`[🔍 Full results and investigation steps](${runUrl})`); - const body = lines.join('\n'); + const staticStatus = staticResult === 'success' + ? 'Passed' + : staticResult === 'failure' + ? 'Failed' + : (staticResult || 'Unknown'); + let evalStatus; + if (!evalRan && !isContributor) { + evalStatus = 'Skipped'; + } else if (discoverResult !== 'success') { + evalStatus = 'Needs attention'; // discovery job failed → can't tell if eval was needed; don't mask as Skipped/Passed + } else if (!hasEntries) { + evalStatus = 'Skipped'; + } else if (hasResults) { + evalStatus = evalPassed ? 'Passed' : 'Failed'; + } else if (evalResult === 'success') { + evalStatus = 'Passed'; + } else if (evalResult === 'failure') { + evalStatus = 'Failed'; + } else if (evalResult === 'skipped') { + evalStatus = 'Skipped'; + } else { + evalStatus = evalResult || 'Unknown'; + } + const overallStatus = staticStatus === 'Failed' || evalStatus === 'Failed' + ? 'Failed' + : (staticStatus === 'Passed' && (evalStatus === 'Passed' || evalStatus === 'Skipped')) + ? 'Passed' + : 'Needs attention'; + const overallIcon = overallStatus === 'Passed' ? '✅' : overallStatus === 'Failed' ? '❌' : '⚠️'; + const badges = [ + badge('Overall', overallStatus, colorFor(overallStatus)), + badge('Static', staticStatus, colorFor(staticStatus)), + badge('LLM', evalStatus, colorFor(evalStatus)), + badge('Skills', String(skillCount), '8250df'), + badge('Agents', String(agentCount), '0969da'), + ].join('\n'); + const authorLine = prAuthor + ? `> @${prAuthor} — new skill validation results are available based on this last commit: ${headSha7}.` + : `> New skill validation results are available based on this last commit: ${headSha7}.`; + const content = lines.join('\n'); + let body = [ + marker, + '', + '## Skill Validation Results', + '', + authorLine, + '> To request a fresh validation after new comments or commits, comment `/evaluate-skills`.', + '', + '

', + badges, + '

', + '', + ``, + '
', + `${overallIcon} Skill Validation Results — ${headSha7} · ${prTitle} · ${timestamp}`, + '
', + '', + content, + '', + '
', + ``, + ].join('\n').replace(//g, '
'); // ── Write step summary with investigation prompt ────── const summaryPath = process.env.GITHUB_STEP_SUMMARY; diff --git a/NuGet.config b/NuGet.config index 752fe77e3232..17aa9aa4f1a1 100644 --- a/NuGet.config +++ b/NuGet.config @@ -4,6 +4,13 @@ + + + + + + + @@ -15,6 +22,12 @@ + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index acff08c44cb7..a19de89c018d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,216 +1,211 @@ - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/android - 01024bb616e7b80417a2c6d320885bfdb956f20a + 1b0756f20835e2e0f4a8f757c0dd77803ca52c45 - + https://github.com/dotnet/android - 1dcfb6f8779c33b6f768c996495cb90ecd729329 + e1d3646df9cb50b2a0924f5b67fa78f9750ae489 - + https://github.com/dotnet/macios - 23eb1c2c9465fe76c810c8a69982c1254161f4b0 + c4595304de2bbb397ff48a95ea05109aacd424a3 - + https://github.com/dotnet/macios - 23eb1c2c9465fe76c810c8a69982c1254161f4b0 + c4595304de2bbb397ff48a95ea05109aacd424a3 - + https://github.com/dotnet/macios - 23eb1c2c9465fe76c810c8a69982c1254161f4b0 + c4595304de2bbb397ff48a95ea05109aacd424a3 - + https://github.com/dotnet/macios - 23eb1c2c9465fe76c810c8a69982c1254161f4b0 + c4595304de2bbb397ff48a95ea05109aacd424a3 - - + https://github.com/dotnet/macios - e5afbf5332820488c4a2d26dad02df88c0110136 + 93d37a419b17d51db04a0f58888a242e0cdfc372 - + https://github.com/dotnet/macios - e5afbf5332820488c4a2d26dad02df88c0110136 + 93d37a419b17d51db04a0f58888a242e0cdfc372 - + https://github.com/dotnet/macios - e5afbf5332820488c4a2d26dad02df88c0110136 + 93d37a419b17d51db04a0f58888a242e0cdfc372 - + https://github.com/dotnet/macios - e5afbf5332820488c4a2d26dad02df88c0110136 + 93d37a419b17d51db04a0f58888a242e0cdfc372 https://dev.azure.com/microsoft/ProjectReunion/_git/ProjectReunionInternal - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 https://github.com/dotnet/templating 3f4da9ced34942d83054e647f3b1d9d7dde281e8 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - - https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 - - - https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 - - + https://github.com/dotnet/xharness - 92962e5c46ac08a66ded4c5696209cc60f1a232f + 31e0b8e08f57890f7b7004b93361d69cd4b21079 - + https://github.com/dotnet/xharness - 92962e5c46ac08a66ded4c5696209cc60f1a232f + 31e0b8e08f57890f7b7004b93361d69cd4b21079 - + https://github.com/dotnet/xharness - 92962e5c46ac08a66ded4c5696209cc60f1a232f + 31e0b8e08f57890f7b7004b93361d69cd4b21079 + + + https://dev.azure.com/dnceng/internal/_git/dotnet-optimization + b194351e54adc9538bb2a0b6a188f8f897fa223c - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 - + https://github.com/dotnet/dotnet - 7b29526f2107416f68578bcb9deaca74fcfcf7f0 + d64191f29ec9042e2696d8b7d8326c4bd10ba268 diff --git a/eng/Versions.props b/eng/Versions.props index f1df08d7d7fe..7b02b6bb27f5 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,14 +1,13 @@ - 10 + 11 0 - 90 - 10.0.100 - ci.main + 0 + 11.0.100 + preview ci.inflight - - + 6 1 @@ -23,59 +22,60 @@ true - 10.0.0.0 + 11.0.0.0 false false - 9.0.120 + 10.0.20 - 10.0.100-rtm.25523.113 + 11.0.100-preview.5.26256.105 $(MicrosoftNETSdkPackageVersion) - 10.0.100 - 10.0.0 + 11.0.0-preview.5.26256.105 $(MicrosoftNETCoreAppRefPackageVersion) $(MicrosoftNETCoreAppRefPackageVersion) $(MicrosoftNETCoreAppRefPackageVersion) + + 1.0.0-prerelease.26153.1 $(MicrosoftNETCoreAppRefPackageVersion) + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 10.3.0 10.3.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 + 11.0.0-preview.2.26103.111 1.0.0-rc2 1.0.0-rc2 1.0.0-rc2 1.0.0-preview.260225.1 - 36.1.2 - 35.0.105 - $(MicrosoftNETSdkAndroidManifest90100PackageVersion) - - 26.0.11017 - 26.0.11017 - 26.0.11017 - 26.0.11017 - - 26.0.9766 - 26.0.9766 - 26.0.9766 - 26.0.9766 + 36.99.0-ci.main.186 + 36.1.53 + $(MicrosoftNetSdkAndroidManifest100100PackageVersion) + + 26.4.11555-net11-p5 + 26.4.11555-net11-p5 + 26.4.11555-net11-p5 + 26.4.11555-net11-p5 + + 26.4.10260 + 26.4.10260 + 26.4.10260 + 26.4.10260 8.0.148 @@ -84,21 +84,21 @@ 1.3.2 1.0.3179.45 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 - 10.0.0 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 + 11.0.0-preview.5.26256.105 - 8.0.16 + 10.0.2 $(MicrosoftAspNetCorePackageVersion) $(MicrosoftAspNetCorePackageVersion) $(MicrosoftAspNetCorePackageVersion) @@ -125,7 +125,7 @@ 6.1.2 17.9.5 17.9.5 - 0.5.0 + 0.4.0 @@ -216,15 +218,15 @@ $([System.Text.RegularExpressions.Regex]::Match($(MicrosoftDotnetSdkInternalPackageVersion), `^\d+\.\d+\.\d`))00 $(VersionBand)$([System.Text.RegularExpressions.Regex]::Match($(MicrosoftDotnetSdkInternalPackageVersion), `\-(preview|rc|alpha).\d+`)) - 10.0.100 + $(DotNetVersionBand) $(DotNetVersionBand) $(DotNetVersionBand) $(DotNetVersionBand) $(DotNetVersionBand) 9.0.100 - $(MicrosoftMacCatalystSdknet100_260PackageVersion) - $(MicrosoftmacOSSdknet100_260PackageVersion) - $(MicrosoftiOSSdknet100_260PackageVersion) - $(MicrosofttvOSSdknet100_260PackageVersion) + $(MicrosoftMacCatalystSdknet110_264PackageVersion) + $(MicrosoftmacOSSdknet110_264PackageVersion) + $(MicrosoftiOSSdknet110_264PackageVersion) + $(MicrosofttvOSSdknet110_264PackageVersion) diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 65ed3a8adef0..fc8d618014e0 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -1,7 +1,6 @@ # This script adds internal feeds required to build commits that depend on internal package sources. For instance, -# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. Similarly, -# dotnet-eng-internal and dotnet-tools-internal are added if dotnet-eng and dotnet-tools are present. -# In addition, this script also enables disabled internal Maestro (darc-int*) feeds. +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. # # Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # @@ -174,16 +173,4 @@ foreach ($dotnetVersion in $dotnetVersions) { } } -# Check for dotnet-eng and add dotnet-eng-internal if present -$dotnetEngSource = $sources.SelectSingleNode("add[@key='dotnet-eng']") -if ($dotnetEngSource -ne $null) { - AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "dotnet-eng-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-eng-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password -} - -# Check for dotnet-tools and add dotnet-tools-internal if present -$dotnetToolsSource = $sources.SelectSingleNode("add[@key='dotnet-tools']") -if ($dotnetToolsSource -ne $null) { - AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "dotnet-tools-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password -} - $doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index b2163abbe71b..b97cc536379d 100755 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -1,9 +1,8 @@ #!/usr/bin/env bash # This script adds internal feeds required to build commits that depend on internal package sources. For instance, -# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. Similarly, -# dotnet-eng-internal and dotnet-tools-internal are added if dotnet-eng and dotnet-tools are present. -# In addition, this script also enables disabled internal Maestro (darc-int*) feeds. +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. # # Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # @@ -174,18 +173,6 @@ for DotNetVersion in ${DotNetVersions[@]} ; do fi done -# Check for dotnet-eng and add dotnet-eng-internal if present -grep -i " /dev/null -if [ "$?" == "0" ]; then - AddOrEnablePackageSource "dotnet-eng-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-eng-internal/nuget/$FeedSuffix" -fi - -# Check for dotnet-tools and add dotnet-tools-internal if present -grep -i " /dev/null -if [ "$?" == "0" ]; then - AddOrEnablePackageSource "dotnet-tools-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/$FeedSuffix" -fi - # I want things split line by line PrevIFS=$IFS IFS=$'\n' diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 8cfee107e7a3..18397a60eb85 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -6,6 +6,7 @@ Param( [string][Alias('v')]$verbosity = "minimal", [string] $msbuildEngine = $null, [bool] $warnAsError = $true, + [string] $warnNotAsError = '', [bool] $nodeReuse = $true, [switch] $buildCheck = $false, [switch][Alias('r')]$restore, @@ -70,6 +71,7 @@ function Print-Usage() { Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)" Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + Write-Host " -warnNotAsError Sets a semi-colon delimited list of warning codes that should not be treated as errors" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" diff --git a/eng/common/build.sh b/eng/common/build.sh index 9767bb411a4f..5883e53bcfb1 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -42,6 +42,7 @@ usage() echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + echo " --warnNotAsError Sets a semi-colon delimited list of warning codes that should not be treated as errors" echo " --buildCheck Sets /check msbuild parameter" echo " --fromVMR Set when building from within the VMR" echo "" @@ -78,6 +79,7 @@ ci=false clean=false warn_as_error=true +warn_not_as_error='' node_reuse=true build_check=false binary_log=false @@ -92,7 +94,7 @@ runtime_source_feed='' runtime_source_feed_key='' properties=() -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -help|-h) @@ -176,6 +178,10 @@ while [[ $# > 0 ]]; do warn_as_error=$2 shift ;; + -warnnotaserror) + warn_not_as_error=$2 + shift + ;; -nodereuse) node_reuse=$2 shift diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index 5ce518406198..66c7988f222a 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -19,17 +19,19 @@ parameters: # publishing defaults artifacts: '' enableMicrobuild: false + enablePreviewMicrobuild: false + microbuildPluginVersion: 'latest' enableMicrobuildForMacAndLinux: false microbuildUseESRP: true enablePublishBuildArtifacts: false enablePublishBuildAssets: false enablePublishTestResults: false + enablePublishing: false enableBuildRetry: false mergeTestResults: false testRunTitle: '' testResultsFormat: '' name: '' - componentGovernanceSteps: [] preSteps: [] artifactPublishSteps: [] runAsPublic: false @@ -71,6 +73,8 @@ jobs: templateContext: ${{ parameters.templateContext }} variables: + - name: AllowPtrToDetectTestRunRetryFiles + value: true - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE value: '$(Build.Repository.Uri)' @@ -128,6 +132,8 @@ jobs: - template: /eng/common/core-templates/steps/install-microbuild.yml parameters: enableMicrobuild: ${{ parameters.enableMicrobuild }} + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildPluginVersion: ${{ parameters.microbuildPluginVersion }} enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }} microbuildUseESRP: ${{ parameters.microbuildUseESRP }} continueOnError: ${{ parameters.continueOnError }} @@ -146,13 +152,12 @@ jobs: - ${{ each step in parameters.steps }}: - ${{ step }} - - ${{ each step in parameters.componentGovernanceSteps }}: - - ${{ step }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/common/core-templates/steps/cleanup-microbuild.yml parameters: enableMicrobuild: ${{ parameters.enableMicrobuild }} + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildPluginVersion: ${{ parameters.microbuildPluginVersion }} enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }} continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index c5788829a872..eefed3b667a4 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -52,13 +52,13 @@ jobs: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 + image: windows.vs2026.amd64 os: windows steps: diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 3437087c80fc..700f77114658 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -74,13 +74,13 @@ jobs: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 + image: windows.vs2026.amd64 os: windows steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: @@ -91,8 +91,8 @@ jobs: fetchDepth: 3 clean: true - - ${{ if eq(parameters.isAssetlessBuild, 'false') }}: - - ${{ if eq(parameters.publishingVersion, 3) }}: + - ${{ if eq(parameters.isAssetlessBuild, 'false') }}: + - ${{ if eq(parameters.publishingVersion, 3) }}: - task: DownloadPipelineArtifact@2 displayName: Download Asset Manifests inputs: @@ -117,7 +117,7 @@ jobs: flattenFolders: true condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: NuGetAuthenticate@1 # Populate internal runtime variables. @@ -125,7 +125,7 @@ jobs: ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: parameters: legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) - + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - task: AzureCLI@2 @@ -145,7 +145,7 @@ jobs: condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -172,17 +172,18 @@ jobs: targetPath: '$(Build.ArtifactStagingDirectory)/MergedManifest.xml' artifactName: AssetManifests displayName: 'Publish Merged Manifest' - retryCountOnTaskFailure: 10 # for any logs being locked - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # just metadata for publishing - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} args: displayName: Publish ReleaseConfigs Artifact - pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs' - publishLocation: Container + targetPath: '$(Build.StagingDirectory)/ReleaseConfigs' artifactName: ReleaseConfigs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # just metadata for publishing - ${{ if or(eq(parameters.publishAssetsImmediately, 'true'), eq(parameters.isAssetlessBuild, 'true')) }}: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -190,7 +191,7 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} - + # Darc is targeting 8.0, so make sure it's installed - task: UseDotNet@2 inputs: @@ -218,4 +219,5 @@ jobs: - template: /eng/common/core-templates/steps/publish-logs.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} - JobLabel: 'Publish_Artifacts_Logs' + StageLabel: 'BuildAssetRegistry' + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/core-templates/job/renovate.yml b/eng/common/core-templates/job/renovate.yml new file mode 100644 index 000000000000..ff86c80b4689 --- /dev/null +++ b/eng/common/core-templates/job/renovate.yml @@ -0,0 +1,196 @@ +# -------------------------------------------------------------------------------------- +# Renovate Bot Job Template +# -------------------------------------------------------------------------------------- +# This Azure DevOps pipeline job template runs Renovate (https://docs.renovatebot.com/) +# to automatically update dependencies in a GitHub repository. +# +# Renovate scans the repository for dependency files and creates pull requests to update +# outdated dependencies based on the configuration specified in the renovateConfigPath +# parameter. +# +# Usage: +# For each product repo wanting to make use of Renovate, this template is called from +# an internal Azure DevOps pipeline, typically with a schedule trigger, to check for +# and propose dependency updates. +# +# For more info, see https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md +# -------------------------------------------------------------------------------------- + +parameters: + +# Path to the Renovate configuration file within the repository. +- name: renovateConfigPath + type: string + default: 'eng/renovate.json' + +# GitHub repository to run Renovate against, in the format 'owner/repo'. +# This could technically be any repo but convention is to target the same +# repo that contains the calling pipeline. The Renovate config file would +# be co-located with the pipeline's repo and, in most cases, the config +# file is specific to the repo being targeted. +- name: gitHubRepo + type: string + +# List of base branches to target for Renovate PRs. +# NOTE: The Renovate configuration file is always read from the branch where the +# pipeline is run, NOT from the target branches specified here. If you need different +# configurations for different branches, run the pipeline from each branch separately. +- name: baseBranches + type: object + default: + - main + +# When true, Renovate will run in dry run mode, which previews changes without creating PRs. +# See the 'Run Renovate' step log output for details of what would have been changed. +- name: dryRun + type: boolean + default: false + +# By default, Renovate will not recreate a PR for a given dependency/version pair that was +# previously closed. This allows opting in to always recreating PRs even if they were +# previously closed. +- name: forceRecreatePR + type: boolean + default: false + +# Name of the arcade repository resource in the pipeline. +# This allows repos which haven't been onboarded to Arcade to still use this +# template by checking out the repo as a resource with a custom name and pointing +# this parameter to it. +- name: arcadeRepoResource + type: string + default: self + +# Directory name for the self repo under $(Build.SourcesDirectory) in multi-checkout. +# In multi-checkout (when arcadeRepoResource != 'self'), Azure DevOps checks out the +# self repo to $(Build.SourcesDirectory)/. Set this to match the auto-generated +# directory name. Using the auto-generated name is necessary rather than explicitly +# defining a checkout path because container jobs expect repos to live under the agent's +# workspace ($(Pipeline.Workspace)). On some self-hosted setups the host path +# (e.g., /mnt/vss/_work) differs from the container path (e.g., /__w), and a custom checkout +# path can fail validation. Using the default checkout location keeps the paths consistent +# and avoids this issue. +- name: selfRepoName + type: string + default: '' +- name: arcadeRepoName + type: string + default: '' + +# Pool configuration for the job. +- name: pool + type: object + default: + name: NetCore1ESPool-Internal + image: build.azurelinux.3.amd64 + os: linux + +jobs: +- job: Renovate + displayName: Run Renovate + container: RenovateContainer + variables: + - group: dotnet-renovate-bot + # The Renovate version is automatically updated by https://github.com/dotnet/arcade/blob/main/azure-pipelines-renovate.yml. + # Changing the variable name here would require updating the name in https://github.com/dotnet/arcade/blob/main/eng/renovate.json as well. + - name: renovateVersion + value: '42' + readonly: true + - name: renovateLogFilePath + value: '$(Build.ArtifactStagingDirectory)/renovate.json' + readonly: true + - name: dryRunArg + readonly: true + ${{ if eq(parameters.dryRun, true) }}: + value: 'full' + ${{ else }}: + value: '' + - name: recreateWhenArg + readonly: true + ${{ if eq(parameters.forceRecreatePR, true) }}: + value: 'always' + ${{ else }}: + value: '' + # In multi-checkout (without custom paths), Azure DevOps places each repo under + # $(Build.SourcesDirectory)/. selfRepoName must be provided in that case. + - name: selfRepoPath + readonly: true + ${{ if eq(parameters.arcadeRepoResource, 'self') }}: + value: '$(Build.SourcesDirectory)' + ${{ else }}: + value: '$(Build.SourcesDirectory)/${{ parameters.selfRepoName }}' + - name: arcadeRepoPath + readonly: true + ${{ if eq(parameters.arcadeRepoResource, 'self') }}: + value: '$(Build.SourcesDirectory)' + ${{ else }}: + value: '$(Build.SourcesDirectory)/${{ parameters.arcadeRepoName }}' + pool: ${{ parameters.pool }} + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + displayName: Publish Renovate Log + condition: succeededOrFailed() + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: $(Agent.JobName)_Logs_Attempt$(System.JobAttempt) + isProduction: false # logs are non-production artifacts + + steps: + - checkout: self + fetchDepth: 1 + + - ${{ if ne(parameters.arcadeRepoResource, 'self') }}: + - checkout: ${{ parameters.arcadeRepoResource }} + fetchDepth: 1 + + - script: | + renovate-config-validator $(selfRepoPath)/${{parameters.renovateConfigPath}} 2>&1 | tee /tmp/renovate-config-validator.out + validatorExit=${PIPESTATUS[0]} + if grep -q '^ WARN:' /tmp/renovate-config-validator.out; then + echo "##vso[task.logissue type=warning]Renovate config validator produced warnings." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + exit $validatorExit + displayName: Validate Renovate config + env: + LOG_LEVEL: info + LOG_FILE_LEVEL: debug + LOG_FILE: $(Build.ArtifactStagingDirectory)/renovate-config-validator.json + + - script: | + . $(arcadeRepoPath)/eng/common/renovate.env + renovate 2>&1 | tee /tmp/renovate.out + renovateExit=${PIPESTATUS[0]} + if grep -q '^ WARN:' /tmp/renovate.out; then + echo "##vso[task.logissue type=warning]Renovate produced warnings." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + exit $renovateExit + displayName: Run Renovate + env: + RENOVATE_FORK_TOKEN: $(BotAccount-dotnet-renovate-bot-PAT) + RENOVATE_TOKEN: $(BotAccount-dotnet-renovate-bot-PAT) + RENOVATE_REPOSITORIES: ${{parameters.gitHubRepo}} + RENOVATE_BASE_BRANCHES: ${{ convertToJson(parameters.baseBranches) }} + RENOVATE_DRY_RUN: $(dryRunArg) + RENOVATE_RECREATE_WHEN: $(recreateWhenArg) + LOG_LEVEL: info + LOG_FILE_LEVEL: debug + LOG_FILE: $(renovateLogFilePath) + RENOVATE_CONFIG_FILE: $(selfRepoPath)/${{parameters.renovateConfigPath}} + + - script: | + echo "PRs created by Renovate:" + if [ -s "$(renovateLogFilePath)" ]; then + if ! jq -r 'select(.msg == "PR created" and .pr != null) | "https://github.com/\(.repository)/pull/\(.pr)"' "$(renovateLogFilePath)" | sort -u; then + echo "##vso[task.logissue type=warning]Failed to parse Renovate log file with jq." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + else + echo "##vso[task.logissue type=warning]No Renovate log file found or file is empty." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + displayName: List created PRs + condition: and(succeededOrFailed(), eq('${{ parameters.dryRun }}', false)) diff --git a/eng/common/core-templates/job/source-build.yml b/eng/common/core-templates/job/source-build.yml index d805d5faeb94..1997c2ae00d7 100644 --- a/eng/common/core-templates/job/source-build.yml +++ b/eng/common/core-templates/job/source-build.yml @@ -60,19 +60,19 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals build.ubuntu.2004.amd64 + demands: ImageOverride -equals build.azurelinux.3.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - image: 1es-mariner-2 + image: build.azurelinux.3.amd64 os: linux ${{ else }}: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open + demands: ImageOverride -equals build.azurelinux.3.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 + demands: ImageOverride -equals build.azurelinux.3.amd64 ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index 76baf5c27258..bac6ac5faac3 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -15,6 +15,8 @@ jobs: variables: - name: BinlogPath value: ${{ parameters.binlogPath }} + - name: skipComponentGovernanceDetection + value: true - template: /eng/common/core-templates/variables/pool-providers.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} @@ -25,10 +27,10 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) - image: windows.vs2026preview.scout.amd64.open + image: windows.vs2026.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: windows.vs2026preview.scout.amd64 + image: windows.vs2026.amd64 steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: diff --git a/eng/common/core-templates/jobs/jobs.yml b/eng/common/core-templates/jobs/jobs.yml index 01ada7476651..cc8cce452786 100644 --- a/eng/common/core-templates/jobs/jobs.yml +++ b/eng/common/core-templates/jobs/jobs.yml @@ -43,6 +43,10 @@ parameters: artifacts: {} is1ESPipeline: '' + + # Publishing version w/default. + publishingVersion: 3 + repositoryAlias: self officialBuildId: '' @@ -102,6 +106,7 @@ jobs: parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} continueOnError: ${{ parameters.continueOnError }} + publishingVersion: ${{ parameters.publishingVersion }} dependsOn: - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ each job in parameters.publishBuildAssetsDependsOn }}: diff --git a/eng/common/core-templates/post-build/common-variables.yml b/eng/common/core-templates/post-build/common-variables.yml index d5627a994ae5..db298ae16bae 100644 --- a/eng/common/core-templates/post-build/common-variables.yml +++ b/eng/common/core-templates/post-build/common-variables.yml @@ -11,8 +11,6 @@ variables: - name: MaestroApiVersion value: "2020-02-20" - - name: SourceLinkCLIVersion - value: 3.0.0 - name: SymbolToolVersion value: 1.0.1 - name: BinlogToolVersion diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index 9423d71ca3a2..8aa86e304919 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -1,117 +1,108 @@ parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: requireDefaultChannels - displayName: Fail the build if there are no default channel(s) registrations for the current build - type: boolean - default: false - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - - name: isAssetlessBuild - type: boolean - displayName: Is Assetless Build - default: false - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - - - name: is1ESPipeline - type: boolean - default: false +# Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. +# Publishing V1 is no longer supported +# Publishing V2 is no longer supported +# Publishing V3 is the default +- name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + - 4 + +- name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + +- name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + +- name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + +- name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + +- name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + +- name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + +- name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + +- name: requireDefaultChannels + displayName: Fail the build if there are no default channel(s) registrations for the current build + type: boolean + default: false + +- name: isAssetlessBuild + type: boolean + displayName: Is Assetless Build + default: false + +# These parameters let the user customize the call to sdk-task.ps1 for publishing +# symbols & general artifacts as well as for signing validation +- name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + +- name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + +- name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + +# Which stages should finish execution before post-build stages start +- name: validateDependsOn + type: object + default: + - build + +- name: publishDependsOn + type: object + default: + - Validate + +# Optional: Call asset publishing rather than running in a separate stage +- name: publishAssetsImmediately + type: boolean + default: false + +- name: is1ESPipeline + type: boolean + default: false stages: -- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true')) }}: - stage: Validate dependsOn: ${{ parameters.validateDependsOn }} displayName: Validate Build Assets variables: - - template: /eng/common/core-templates/post-build/common-variables.yml - - template: /eng/common/core-templates/variables/pool-providers.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} jobs: - job: displayName: NuGet Validation @@ -120,26 +111,27 @@ stages: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) - image: windows.vs2026preview.scout.amd64 + image: windows.vs2026.amd64 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026preview.scout.amd64 + demands: ImageOverride -equals windows.vs2026.amd64 steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: - task: DownloadBuildArtifacts@0 displayName: Download Package Artifacts inputs: @@ -150,12 +142,25 @@ stages: buildId: $(AzDOBuildId) artifactName: PackageArtifacts checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - job: displayName: Signing Validation @@ -164,25 +169,26 @@ stages: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: + ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 + image: windows.vs2026.amd64 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026preview.scout.amd64 + demands: ImageOverride -equals windows.vs2026.amd64 steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: - task: DownloadBuildArtifacts@0 displayName: Download Package Artifacts inputs: @@ -193,91 +199,71 @@ stages: buildId: $(AzDOBuildId) artifactName: PackageArtifacts checkDownloadedFiles: true - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: /eng/common/core-templates/steps/publish-logs.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - StageLabel: 'Validation' - JobLabel: 'Signing' - BinlogToolVersion: $(BinlogToolVersion) - - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026preview.scout.amd64 - steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine dotnet + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + # SourceLink validation has been removed — the underlying CLI tool + # (targeting netcoreapp2.1) has not functioned for years. + # The enableSourceLinkValidation parameter is kept but ignored so + # existing pipelines that pass it are not broken. + # See https://github.com/dotnet/arcade/issues/16647 + - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: + - job: + displayName: 'SourceLink Validation Removed - please remove enableSourceLinkValidation from your pipeline' + pool: server + steps: + - task: Delay@1 + displayName: 'Warning: SourceLink validation removed (see https://github.com/dotnet/arcade/issues/16647)' inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true + delayForMinutes: '0' - ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - stage: publish_using_darc - ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true')) }}: dependsOn: ${{ parameters.publishDependsOn }} ${{ else }}: dependsOn: ${{ parameters.validateDependsOn }} displayName: Publish using Darc variables: - - template: /eng/common/core-templates/post-build/common-variables.yml - - template: /eng/common/core-templates/variables/pool-providers.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} jobs: - job: displayName: Publish Using Darc @@ -286,49 +272,48 @@ stages: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: + ${{ if eq(parameters.is1ESPipeline, true) }}: name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 + image: windows.vs2026.amd64 os: windows ${{ else }}: name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2019.amd64 + demands: ImageOverride -equals windows.vs2026.amd64 steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - task: NuGetAuthenticate@1 - - # Populate internal runtime variables. - - template: /eng/common/templates/steps/enable-internal-sources.yml - parameters: - legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) - - - template: /eng/common/templates/steps/enable-internal-runtimes.yml + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} - # Darc is targeting 8.0, so make sure it's installed - - task: UseDotNet@2 - inputs: - version: 8.0.x + - task: NuGetAuthenticate@1 - - task: AzureCLI@2 - displayName: Publish Using Darc - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: > + # Populate internal runtime variables. + - template: /eng/common/templates/steps/enable-internal-sources.yml + parameters: + legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) + + - template: /eng/common/templates/steps/enable-internal-runtimes.yml + + - task: UseDotNet@2 + inputs: + version: 8.0.x + + - task: AzureCLI@2 + displayName: Publish Using Darc + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: > -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -PublishingInfraVersion 3 -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -RequireDefaultChannels ${{ parameters.requireDefaultChannels }} diff --git a/eng/common/core-templates/post-build/setup-maestro-vars.yml b/eng/common/core-templates/post-build/setup-maestro-vars.yml index a7abd58c4bb6..6dfa99ec5e37 100644 --- a/eng/common/core-templates/post-build/setup-maestro-vars.yml +++ b/eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -8,12 +8,11 @@ steps: - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@2 displayName: Download Release Configs inputs: - buildType: current artifactName: ReleaseConfigs - checkDownloadedFiles: true + targetPath: '$(Build.StagingDirectory)/ReleaseConfigs' - task: AzureCLI@2 name: setReleaseVars diff --git a/eng/common/core-templates/stages/renovate.yml b/eng/common/core-templates/stages/renovate.yml new file mode 100644 index 000000000000..edab28182585 --- /dev/null +++ b/eng/common/core-templates/stages/renovate.yml @@ -0,0 +1,111 @@ +# -------------------------------------------------------------------------------------- +# Renovate Pipeline Template +# -------------------------------------------------------------------------------------- +# This template provides a complete reusable pipeline definition for running Renovate +# in a 1ES Official pipeline. Pipelines can extend from this template and only need +# to pass the Renovate job parameters. +# +# For more info, see https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md +# -------------------------------------------------------------------------------------- + +parameters: + +# Path to the Renovate configuration file within the repository. +- name: renovateConfigPath + type: string + default: 'eng/renovate.json' + +# GitHub repository to run Renovate against, in the format 'owner/repo'. +- name: gitHubRepo + type: string + +# List of base branches to target for Renovate PRs. +- name: baseBranches + type: object + default: + - main + +# When true, Renovate will run in dry run mode. +- name: dryRun + type: boolean + default: false + +# When true, Renovate will recreate PRs even if they were previously closed. +- name: forceRecreatePR + type: boolean + default: false + +# Name of the arcade repository resource in the pipeline. +# This allows repos which haven't been onboarded to Arcade to still use this +# template by checking out the repo as a resource with a custom name and pointing +# this parameter to it. +- name: arcadeRepoResource + type: string + default: 'self' + +- name: selfRepoName + type: string + default: '' +- name: arcadeRepoName + type: string + default: '' + +# Pool configuration for the pipeline. +- name: pool + type: object + default: + name: NetCore1ESPool-Internal + image: build.azurelinux.3.amd64 + os: linux + +# Renovate version used in the container image tag. +- name: renovateVersion + default: 43 + type: number + +# Pool configuration for SDL analysis. +- name: sdlPool + type: object + default: + name: NetCore1ESPool-Internal + image: windows.vs2026.amd64 + os: windows + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: ${{ parameters.pool }} + sdl: + sourceAnalysisPool: ${{ parameters.sdlPool }} + # When repos that aren't onboarded to Arcade use this template, they set the + # arcadeRepoResource parameter to point to their Arcade repo resource. In that case, + # Aracde will be excluded from SDL analysis. + ${{ if ne(parameters.arcadeRepoResource, 'self') }}: + sourceRepositoriesToScan: + exclude: + - repository: ${{ parameters.arcadeRepoResource }} + containers: + RenovateContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-renovate-${{ parameters.renovateVersion }}-amd64 + stages: + - stage: Renovate + displayName: Run Renovate + jobs: + - template: /eng/common/core-templates/job/renovate.yml@${{ parameters.arcadeRepoResource }} + parameters: + renovateConfigPath: ${{ parameters.renovateConfigPath }} + gitHubRepo: ${{ parameters.gitHubRepo }} + baseBranches: ${{ parameters.baseBranches }} + dryRun: ${{ parameters.dryRun }} + forceRecreatePR: ${{ parameters.forceRecreatePR }} + pool: ${{ parameters.pool }} + arcadeRepoResource: ${{ parameters.arcadeRepoResource }} + selfRepoName: ${{ parameters.selfRepoName }} + arcadeRepoName: ${{ parameters.arcadeRepoName }} diff --git a/eng/common/core-templates/steps/generate-sbom.yml b/eng/common/core-templates/steps/generate-sbom.yml index c05f65027979..aad0a8aeda33 100644 --- a/eng/common/core-templates/steps/generate-sbom.yml +++ b/eng/common/core-templates/steps/generate-sbom.yml @@ -1,54 +1,14 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - parameters: - PackageVersion: 10.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - is1ESPipeline: false - # disable publishArtifacts if some other step is publishing the artifacts (like job.yml). - publishArtifacts: true + PackageVersion: unused + BuildDropPath: unused + PackageName: unused + ManifestDirPath: unused + IgnoreDirectories: unused + sbomContinueOnError: unused + is1ESPipeline: unused + publishArtifacts: unused steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 - script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }}/$(ARTIFACT_NAME) - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- ${{ if eq(parameters.publishArtifacts, 'true')}}: - - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - args: - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - targetPath: '${{ parameters.manifestDirPath }}' - artifactName: $(ARTIFACT_NAME) - + echo "##vso[task.logissue type=warning]Including generate-sbom.yml is deprecated, SBOM generation is handled 1ES PT now. Remove this include." + displayName: Issue generate-sbom.yml deprecation warning diff --git a/eng/common/core-templates/steps/install-microbuild-impl.yml b/eng/common/core-templates/steps/install-microbuild-impl.yml new file mode 100644 index 000000000000..da22beb3f60c --- /dev/null +++ b/eng/common/core-templates/steps/install-microbuild-impl.yml @@ -0,0 +1,34 @@ +parameters: + - name: microbuildTaskInputs + type: object + default: {} + + - name: microbuildEnv + type: object + default: {} + + - name: enablePreviewMicrobuild + type: boolean + default: false + + - name: condition + type: string + + - name: continueOnError + type: boolean + +steps: +- ${{ if eq(parameters.enablePreviewMicrobuild, true) }}: + - task: MicroBuildSigningPluginPreview@4 + displayName: Install Preview MicroBuild plugin + inputs: ${{ parameters.microbuildTaskInputs }} + env: ${{ parameters.microbuildEnv }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} +- ${{ else }}: + - task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild plugin + inputs: ${{ parameters.microbuildTaskInputs }} + env: ${{ parameters.microbuildEnv }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} diff --git a/eng/common/core-templates/steps/install-microbuild.yml b/eng/common/core-templates/steps/install-microbuild.yml index 553fce66b940..76a54e157fda 100644 --- a/eng/common/core-templates/steps/install-microbuild.yml +++ b/eng/common/core-templates/steps/install-microbuild.yml @@ -4,6 +4,8 @@ parameters: # Enable install tasks for MicroBuild on Mac and Linux # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT' enableMicrobuildForMacAndLinux: false + # Enable preview version of MB signing plugin + enablePreviewMicrobuild: false # Determines whether the ESRP service connection information should be passed to the signing plugin. # This overlaps with _SignType to some degree. We only need the service connection for real signing. # It's important that the service connection not be passed to the MicroBuildSigningPlugin task in this place. @@ -13,6 +15,8 @@ parameters: microbuildUseESRP: true # Microbuild installation directory microBuildOutputFolder: $(Agent.TempDirectory)/MicroBuild + # Microbuild version + microbuildPluginVersion: 'latest' continueOnError: false @@ -69,42 +73,46 @@ steps: # YAML expansion, and Windows vs. Linux/Mac uses different service connections. However, # we can avoid including the MB install step if not enabled at all. This avoids a bunch of # extra pipeline authorizations, since most pipelines do not sign on non-Windows. - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin (Windows) - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - ${{ if eq(parameters.microbuildUseESRP, true) }}: - ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea - ${{ else }}: - ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca - env: - TeamName: $(_TeamName) - MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'), in(variables['_SignType'], 'real', 'test')) - - - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, true) }}: - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin (non-Windows) - inputs: + - template: /eng/common/core-templates/steps/install-microbuild-impl.yml + parameters: + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildTaskInputs: signType: $(_SignType) zipSources: false feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - workingDirectory: ${{ parameters.microBuildOutputFolder }} + version: ${{ parameters.microbuildPluginVersion }} ${{ if eq(parameters.microbuildUseESRP, true) }}: ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - ConnectedPMEServiceName: beb8cb23-b303-4c95-ab26-9e44bc958d39 + ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea ${{ else }}: - ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc - env: + ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca + microbuildEnv: TeamName: $(_TeamName) MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'), eq(variables['_SignType'], 'real')) + condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'), in(variables['_SignType'], 'real', 'test')) + + - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, true) }}: + - template: /eng/common/core-templates/steps/install-microbuild-impl.yml + parameters: + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildTaskInputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + version: ${{ parameters.microbuildPluginVersion }} + workingDirectory: ${{ parameters.microBuildOutputFolder }} + ${{ if eq(parameters.microbuildUseESRP, true) }}: + ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + ConnectedPMEServiceName: beb8cb23-b303-4c95-ab26-9e44bc958d39 + ${{ else }}: + ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc + microbuildEnv: + TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'), eq(variables['_SignType'], 'real')) diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml index 5a927b4c7bcb..84a1922c73f3 100644 --- a/eng/common/core-templates/steps/publish-logs.yml +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -31,7 +31,6 @@ steps: -runtimeSourceFeed https://ci.dot.net/internal -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' '$(publishing-dnceng-devdiv-code-r-build-re)' - '$(MaestroAccessToken)' '$(dn-bot-all-orgs-artifact-feeds-rw)' '$(akams-client-id)' '$(microsoft-symbol-server-pat)' @@ -51,13 +50,15 @@ steps: TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' condition: always() -- template: /eng/common/core-templates/steps/publish-build-artifacts.yml +- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} args: displayName: Publish Logs - pathToPublish: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' - publishLocation: Container - artifactName: PostBuildLogs + targetPath: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' + artifactName: PostBuildLogs_${{ parameters.StageLabel }}_${{ parameters.JobLabel }}_Attempt$(System.JobAttempt) continueOnError: true condition: always() + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts + diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml index b9c86c18ae42..b75f59c428d4 100644 --- a/eng/common/core-templates/steps/source-build.yml +++ b/eng/common/core-templates/steps/source-build.yml @@ -24,7 +24,7 @@ steps: # in the default public locations. internalRuntimeDownloadArgs= if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey '$(dotnetbuilds-internal-container-read-token-base64)'' + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' fi buildConfig=Release @@ -62,4 +62,4 @@ steps: artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) continueOnError: true condition: succeededOrFailed() - sbomEnabled: false # we don't need SBOM for logs + isProduction: false # logs are non-production artifacts diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index e9a694afa58e..3ad83b8c3075 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -1,6 +1,6 @@ parameters: - sourceIndexUploadPackageVersion: 2.0.0-20250818.1 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20250818.1 + sourceIndexUploadPackageVersion: 2.0.0-20250906.1 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20250906.1 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json binlogPath: artifacts/log/Debug/Build.binlog @@ -14,8 +14,8 @@ steps: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: "Source Index: Download netsourceindex Tools" # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) diff --git a/eng/common/cross/arm/tizen/tizen.patch b/eng/common/cross/arm/tizen/tizen.patch new file mode 100644 index 000000000000..fb12ade7250a --- /dev/null +++ b/eng/common/cross/arm/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-littlearm) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-armhf.so.3 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-armhf.so.3 ) ) diff --git a/eng/common/cross/arm64/tizen/tizen.patch b/eng/common/cross/arm64/tizen/tizen.patch new file mode 100644 index 000000000000..2cebc547382e --- /dev/null +++ b/eng/common/cross/arm64/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib64/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib64/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf64-littleaarch64) +-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-aarch64.so.1 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-aarch64.so.1 ) ) diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 8abfb71f7275..314c93c57598 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -9,6 +9,7 @@ usage() echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine" echo " for alpine can be specified with version: alpineX.YY or alpineedge" echo " for FreeBSD can be: freebsd13, freebsd14" + echo " for OpenBSD can be: openbsd" echo " for illumos can be: illumos" echo " for Haiku can be: haiku." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" @@ -27,6 +28,8 @@ __BuildArch=arm __AlpineArch=armv7 __FreeBSDArch=arm __FreeBSDMachineArch=armv7 +__OpenBSDArch=arm +__OpenBSDMachineArch=armv7 __IllumosArch=arm7 __HaikuArch=arm __QEMUArch=arm @@ -72,7 +75,7 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="13.4-RELEASE" +__FreeBSDBase="13.5-RELEASE" __FreeBSDPkg="1.21.3" __FreeBSDABI="13" __FreeBSDPackages="libunwind" @@ -82,6 +85,12 @@ __FreeBSDPackages+=" openssl" __FreeBSDPackages+=" krb5" __FreeBSDPackages+=" terminfo-db" +__OpenBSDVersion="7.8" +__OpenBSDPackages="heimdal-libs" +__OpenBSDPackages+=" icu4c" +__OpenBSDPackages+=" inotify-tools" +__OpenBSDPackages+=" openssl" + __IllumosPackages="icu" __IllumosPackages+=" mit-krb5" __IllumosPackages+=" openssl" @@ -160,6 +169,8 @@ while :; do __QEMUArch=aarch64 __FreeBSDArch=arm64 __FreeBSDMachineArch=aarch64 + __OpenBSDArch=arm64 + __OpenBSDMachineArch=aarch64 ;; armel) __BuildArch=armel @@ -235,6 +246,8 @@ while :; do __UbuntuArch=amd64 __FreeBSDArch=amd64 __FreeBSDMachineArch=amd64 + __OpenBSDArch=amd64 + __OpenBSDMachineArch=amd64 __illumosArch=x86_64 __HaikuArch=x86_64 __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" @@ -295,9 +308,7 @@ while :; do ;; noble) # Ubuntu 24.04 __CodeName=noble - if [[ -z "$__LLDB_Package" ]]; then - __LLDB_Package="liblldb-19-dev" - fi + __LLDB_Package="liblldb-19-dev" ;; stretch) # Debian 9 __CodeName=stretch @@ -383,10 +394,14 @@ while :; do ;; freebsd14) __CodeName=freebsd - __FreeBSDBase="14.2-RELEASE" + __FreeBSDBase="14.3-RELEASE" __FreeBSDABI="14" __SkipUnmount=1 ;; + openbsd) + __CodeName=openbsd + __SkipUnmount=1 + ;; illumos) __CodeName=illumos __SkipUnmount=1 @@ -595,6 +610,62 @@ elif [[ "$__CodeName" == "freebsd" ]]; then INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update # shellcheck disable=SC2086 INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages +elif [[ "$__CodeName" == "openbsd" ]]; then + # determine mirrors + OPENBSD_MIRROR="https://cdn.openbsd.org/pub/OpenBSD/$__OpenBSDVersion/$__OpenBSDMachineArch" + + # download base system sets + ensureDownloadTool + + BASE_SETS=(base comp) + for set in "${BASE_SETS[@]}"; do + FILE="${set}${__OpenBSDVersion//./}.tgz" + echo "Downloading $FILE..." + if [[ "$__hasWget" == 1 ]]; then + wget -O- "$OPENBSD_MIRROR/$FILE" | tar -C "$__RootfsDir" -xzpf - + else + curl -SL "$OPENBSD_MIRROR/$FILE" | tar -C "$__RootfsDir" -xzpf - + fi + done + + PKG_MIRROR="https://cdn.openbsd.org/pub/OpenBSD/${__OpenBSDVersion}/packages/${__OpenBSDMachineArch}" + + echo "Installing packages into sysroot..." + + # Fetch package index once + if [[ "$__hasWget" == 1 ]]; then + PKG_INDEX=$(wget -qO- "$PKG_MIRROR/") + else + PKG_INDEX=$(curl -s "$PKG_MIRROR/") + fi + + for pkg in $__OpenBSDPackages; do + PKG_FILE=$(echo "$PKG_INDEX" | grep -Po ">\K${pkg}-[0-9][^\" ]*\.tgz" \ + | sort -V | tail -n1) + + echo "Resolved package filename for $pkg: $PKG_FILE" + + [[ -z "$PKG_FILE" ]] && { echo "ERROR: Package $pkg not found"; exit 1; } + + if [[ "$__hasWget" == 1 ]]; then + wget -O- "$PKG_MIRROR/$PKG_FILE" | tar -C "$__RootfsDir" -xzpf - + else + curl -SL "$PKG_MIRROR/$PKG_FILE" | tar -C "$__RootfsDir" -xzpf - + fi + done + + echo "Creating versionless symlinks for shared libraries..." + # Find all versioned .so files and create the base .so symlink + for lib in "$__RootfsDir/usr/lib/libc++.so."* "$__RootfsDir/usr/lib/libc++abi.so."* "$__RootfsDir/usr/lib/libpthread.so."*; do + if [ -f "$lib" ]; then + # Extract the filename (e.g., libc++.so.12.0) + VERSIONED_NAME=$(basename "$lib") + # Remove the trailing version numbers (e.g., libc++.so) + BASE_NAME=${VERSIONED_NAME%.so.*}.so + # Create the symlink in the same directory + ln -sf "$VERSIONED_NAME" "$__RootfsDir/usr/lib/$BASE_NAME" + fi + done elif [[ "$__CodeName" == "illumos" ]]; then mkdir "$__RootfsDir/tmp" pushd "$__RootfsDir/tmp" diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 0ff85cf0367e..ff2dfdb4a5bf 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -3,15 +3,22 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) # reset platform variables (e.g. cmake 3.25 sets LINUX=1) unset(LINUX) unset(FREEBSD) +unset(OPENBSD) unset(ILLUMOS) unset(ANDROID) unset(TIZEN) unset(HAIKU) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) + +file(GLOB OPENBSD_PROBE "${CROSS_ROOTFS}/etc/signify/openbsd-*.pub") + if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) set(CMAKE_SYSTEM_NAME FreeBSD) set(FREEBSD 1) +elseif(OPENBSD_PROBE) + set(CMAKE_SYSTEM_NAME OpenBSD) + set(OPENBSD 1) elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) set(CMAKE_SYSTEM_NAME SunOS) set(ILLUMOS 1) @@ -53,6 +60,8 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") endif() elseif(FREEBSD) set(triple "aarch64-unknown-freebsd12") + elseif(OPENBSD) + set(triple "aarch64-unknown-openbsd") endif() elseif(TARGET_ARCH_NAME STREQUAL "armel") set(CMAKE_SYSTEM_PROCESSOR armv7l) @@ -109,6 +118,8 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") endif() elseif(FREEBSD) set(triple "x86_64-unknown-freebsd12") + elseif(OPENBSD) + set(triple "x86_64-unknown-openbsd") elseif(ILLUMOS) set(TOOLCHAIN "x86_64-illumos") elseif(HAIKU) @@ -193,7 +204,7 @@ if(ANDROID) # include official NDK toolchain script include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) -elseif(FREEBSD) +elseif(FREEBSD OR OPENBSD) # we cross-compile by instructing clang set(CMAKE_C_COMPILER_TARGET ${triple}) set(CMAKE_CXX_COMPILER_TARGET ${triple}) @@ -291,7 +302,7 @@ endif() # Specify compile options -if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|loongarch64|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) +if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|loongarch64|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD AND NOT OPENBSD) OR ILLUMOS OR HAIKU) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) diff --git a/eng/common/cross/x64/tizen/tizen.patch b/eng/common/cross/x64/tizen/tizen.patch new file mode 100644 index 000000000000..56fbc881095b --- /dev/null +++ b/eng/common/cross/x64/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib64/libc.so b/usr/lib64/libc.so +--- a/usr/lib64/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib64/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf64-x86-64) +-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-x86-64.so.2 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-x86-64.so.2 ) ) diff --git a/eng/common/cross/x86/tizen/tizen.patch b/eng/common/cross/x86/tizen/tizen.patch new file mode 100644 index 000000000000..f4fe8838ad66 --- /dev/null +++ b/eng/common/cross/x86/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-i386) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.2 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.2 ) ) diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index e33743105635..a5be41db6906 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -29,11 +29,11 @@ function InstallDarcCli ($darcVersion, $toolpath) { Write-Host "Installing Darc CLI version $darcVersion..." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' if (-not $toolpath) { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity -g" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g }else { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" } } diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index e889f439b8dc..b56d40e5706c 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -5,7 +5,7 @@ darcVersion='' versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2020-02-20' verbosity='minimal' -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in --darcversion) @@ -73,9 +73,9 @@ function InstallDarcCli { echo "Installing Darc CLI version $darcVersion..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." if [ -z "$toolpath" ]; then - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g) else - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") fi } diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 7b9d97e3bd4d..61f302bb6775 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -18,7 +18,7 @@ architecture='' runtime='dotnet' runtimeSourceFeed='' runtimeSourceFeedKey='' -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in -version|-v) diff --git a/eng/common/dotnet.sh b/eng/common/dotnet.sh index 2ef68235675f..f6d24871c1d4 100644 --- a/eng/common/dotnet.sh +++ b/eng/common/dotnet.sh @@ -19,7 +19,7 @@ source $scriptroot/tools.sh InitializeDotNetCli true # install # Invoke acquired SDK with args if they are provided -if [[ $# > 0 ]]; then +if [[ $# -gt 0 ]]; then __dotnetDir=${_InitializeDotNetCli} dotnetPath=${__dotnetDir}/dotnet ${dotnetPath} "$@" diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 index 92b77347d990..c282d3ae403a 100644 --- a/eng/common/internal-feed-operations.ps1 +++ b/eng/common/internal-feed-operations.ps1 @@ -26,7 +26,7 @@ function SetupCredProvider { $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." - Invoke-WebRequest $url -OutFile installcredprovider.ps1 + Invoke-WebRequest $url -UseBasicParsing -OutFile installcredprovider.ps1 Write-Host 'Installing plugin...' .\installcredprovider.ps1 -Force diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index 9378223ba095..6299e7effd4c 100755 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -100,7 +100,7 @@ operation='' authToken='' repoName='' -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in --operation) diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh index 83ea7aab0e08..8fc6d2fec78d 100644 --- a/eng/common/native/init-distro-rid.sh +++ b/eng/common/native/init-distro-rid.sh @@ -39,6 +39,8 @@ getNonPortableDistroRid() # $rootfsDir can be empty. freebsd-version is a shell script and should always work. __freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1) nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" + elif [ "$targetOs" = "openbsd" ]; then + nonPortableRid="openbsd.$(uname -r)-${targetArch}" elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then __android_sdk_version=$(getprop ro.build.version.sdk) nonPortableRid="android.$__android_sdk_version-${targetArch}" diff --git a/eng/common/native/install-dependencies.sh b/eng/common/native/install-dependencies.sh index 477a44f335be..4742177a7685 100644 --- a/eng/common/native/install-dependencies.sh +++ b/eng/common/native/install-dependencies.sh @@ -24,14 +24,16 @@ case "$os" in apt update apt install -y build-essential gettext locales cmake llvm clang lld lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \ - libssl-dev libkrb5-dev pigz cpio + libssl-dev libkrb5-dev pigz cpio ninja-build localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 - elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ]; then + elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ] || [ "$ID" = "centos" ]; then pkg_mgr="$(command -v tdnf 2>/dev/null || command -v dnf)" - $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio + $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build + elif [ "$ID" = "amzn" ]; then + dnf install -y cmake llvm lld lldb clang python libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build elif [ "$ID" = "alpine" ]; then - apk add build-base cmake bash curl clang llvm-dev lld lldb krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio + apk add build-base cmake bash curl clang llvm llvm-dev lld lldb-dev krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio ninja else echo "Unsupported distro. distro: $ID" exit 1 @@ -52,6 +54,7 @@ brew "openssl@3" brew "pkgconf" brew "python3" brew "pigz" +brew "ninja" EOF ;; diff --git a/eng/common/post-build/nuget-verification.ps1 b/eng/common/post-build/nuget-verification.ps1 index ac5c69ffcac5..eea88e653c91 100644 --- a/eng/common/post-build/nuget-verification.ps1 +++ b/eng/common/post-build/nuget-verification.ps1 @@ -65,7 +65,7 @@ if ($NuGetExePath) { Write-Host "Downloading nuget.exe from $nugetExeUrl..." $ProgressPreference = 'SilentlyContinue' try { - Invoke-WebRequest $nugetExeUrl -OutFile $downloadedNuGetExe + Invoke-WebRequest $nugetExeUrl -UseBasicParsing -OutFile $downloadedNuGetExe $ProgressPreference = 'Continue' } catch { $ProgressPreference = 'Continue' diff --git a/eng/common/post-build/redact-logs.ps1 b/eng/common/post-build/redact-logs.ps1 index 472d5bb562c9..672f4e2652ed 100644 --- a/eng/common/post-build/redact-logs.ps1 +++ b/eng/common/post-build/redact-logs.ps1 @@ -9,7 +9,8 @@ param( [Parameter(Mandatory=$false)][string] $TokensFilePath, [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact, [Parameter(Mandatory=$false)][string] $runtimeSourceFeed, - [Parameter(Mandatory=$false)][string] $runtimeSourceFeedKey) + [Parameter(Mandatory=$false)][string] $runtimeSourceFeedKey +) try { $ErrorActionPreference = 'Stop' @@ -48,8 +49,8 @@ try { Write-Host "Installing Binlog redactor CLI..." Write-Host "'$dotnet' new tool-manifest" & "$dotnet" new tool-manifest - Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" - & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion + Write-Host "'$dotnet' tool install $packageName --local --source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" + & "$dotnet" tool install $packageName --local --source "$PackageFeed" -v $verbosity --version $BinlogToolVersion if (Test-Path $TokensFilePath) { Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath diff --git a/eng/common/renovate.env b/eng/common/renovate.env new file mode 100644 index 000000000000..17ecc05d9b19 --- /dev/null +++ b/eng/common/renovate.env @@ -0,0 +1,42 @@ +# Renovate Global Configuration +# https://docs.renovatebot.com/self-hosted-configuration/ +# +# NOTE: This file uses bash/shell format and is sourced via `. renovate.env`. +# Values containing spaces or special characters must be quoted. + +# Author to use for git commits made by Renovate +# https://docs.renovatebot.com/configuration-options/#gitauthor +export RENOVATE_GIT_AUTHOR='.NET Renovate ' + +# Disable rate limiting for PR creation (0 = unlimited) +# https://docs.renovatebot.com/presets-default/#prhourlylimitnone +# https://docs.renovatebot.com/presets-default/#prconcurrentlimitnone +export RENOVATE_PR_HOURLY_LIMIT=0 +export RENOVATE_PR_CONCURRENT_LIMIT=0 + +# Skip the onboarding PR that Renovate normally creates for new repos +# https://docs.renovatebot.com/config-overview/#onboarding +export RENOVATE_ONBOARDING=false + +# Any Renovate config file in the cloned repository is ignored. Only +# the Renovate config file from the repo where the pipeline is running +# is used (yes, those are the same repo but the sources may be different). +# https://docs.renovatebot.com/self-hosted-configuration/#requireconfig +export RENOVATE_REQUIRE_CONFIG=ignored + +# Customize the PR body content. This removes some of the default +# sections that aren't relevant in a self-hosted config. +# https://docs.renovatebot.com/configuration-options/#prheader +# https://docs.renovatebot.com/configuration-options/#prbodynotes +# https://docs.renovatebot.com/configuration-options/#prbodytemplate +export RENOVATE_PR_HEADER='## Automated Dependency Update' +export RENOVATE_PR_BODY_NOTES='["This PR has been created automatically by the [.NET Renovate Bot](https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md) to update one or more dependencies in your repo. Please review the changes and merge the PR if everything looks good."]' +export RENOVATE_PR_BODY_TEMPLATE='{{{header}}}{{{table}}}{{{warnings}}}{{{notes}}}{{{changelogs}}}' + +# Extend the global config with additional presets +# https://docs.renovatebot.com/self-hosted-configuration/#globalextends +# Disable the Dependency Dashboard issue that tracks all updates +export RENOVATE_GLOBAL_EXTENDS='[":disableDependencyDashboard"]' + +# Allow all commands for post-upgrade commands. +export RENOVATE_ALLOWED_COMMANDS='[".*"]' diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index b64b66a6275b..68119de603ef 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -22,7 +22,7 @@ $warnAsError = if ($noWarnAsError) { $false } else { $true } function Print-Usage() { Write-Host "Common settings:" - Write-Host " -task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)" + Write-Host " -task Name of Arcade task (name of a project in toolset directory of the Arcade SDK package)" Write-Host " -restore Restore dependencies" Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" Write-Host " -help Print help and exit" @@ -66,20 +66,7 @@ try { if( $msbuildEngine -eq "vs") { # Ensure desktop MSBuild is available for sdk tasks. - if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) { - $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty - } - if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "18.0.0" -MemberType NoteProperty - } - if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { - $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true - } - if ($xcopyMSBuildToolsFolder -eq $null) { - throw 'Unable to get xcopy downloadable version of msbuild' - } - - $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe" + $global:_MSBuildExe = InitializeVisualStudioMSBuild } $taskProject = GetSdkTaskProject $task diff --git a/eng/common/sdk-task.sh b/eng/common/sdk-task.sh index 3270f83fa9a7..1cf71bb2aea4 100644 --- a/eng/common/sdk-task.sh +++ b/eng/common/sdk-task.sh @@ -2,7 +2,7 @@ show_usage() { echo "Common settings:" - echo " --task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)" + echo " --task Name of Arcade task (name of a project in toolset directory of the Arcade SDK package)" echo " --restore Restore dependencies" echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" echo " --help Print help and exit" diff --git a/eng/common/template-guidance.md b/eng/common/template-guidance.md index 4bf4cf41bd7c..f772aa3d78fa 100644 --- a/eng/common/template-guidance.md +++ b/eng/common/template-guidance.md @@ -71,7 +71,6 @@ eng\common\ source-build.yml (shim) source-index-stage1.yml (shim) jobs\ - codeql-build.yml (shim) jobs.yml (shim) source-build.yml (shim) post-build\ @@ -82,14 +81,12 @@ eng\common\ publish-build-artifacts.yml (logic) publish-pipeline-artifacts.yml (logic) component-governance.yml (shim) - generate-sbom.yml (shim) publish-logs.yml (shim) retain-build.yml (shim) send-to-helix.yml (shim) source-build.yml (shim) variables\ pool-providers.yml (logic + redirect) # templates/variables/pool-providers.yml will redirect to templates-official/variables/pool-providers.yml if you are running in the internal project - sdl-variables.yml (logic) core-templates\ job\ job.yml (logic) @@ -98,7 +95,6 @@ eng\common\ source-build.yml (logic) source-index-stage1.yml (logic) jobs\ - codeql-build.yml (logic) jobs.yml (logic) source-build.yml (logic) post-build\ @@ -107,7 +103,6 @@ eng\common\ setup-maestro-vars.yml (logic) steps\ component-governance.yml (logic) - generate-sbom.yml (logic) publish-build-artifacts.yml (redirect) publish-logs.yml (logic) publish-pipeline-artifacts.yml (redirect) diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 92a0664f5647..d68e9fbc2656 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -1,24 +1,15 @@ parameters: -# Sbom related params - enableSbom: true runAsPublic: false - PackageVersion: 9.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' +# Sbom related params, unused now and can eventually be removed + enableSbom: unused + PackageVersion: unused + BuildDropPath: unused jobs: - template: /eng/common/core-templates/job/job.yml parameters: is1ESPipeline: true - componentGovernanceSteps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion }} - BuildDropPath: ${{ parameters.buildDropPath }} - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - publishArtifacts: false - # publish artifacts # for 1ES managed templates, use the templateContext.output to handle multiple outputs. templateContext: @@ -26,12 +17,19 @@ jobs: outputs: - ${{ if ne(parameters.artifacts.publish, '') }}: - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - output: buildArtifacts + - output: pipelineArtifact displayName: Publish pipeline artifacts - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + condition: succeeded() + retryCountOnTaskFailure: 10 # for any files being locked + continueOnError: true + - output: pipelineArtifact + displayName: Publish pipeline artifacts + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}_Attempt$(System.JobAttempt) + condition: not(succeeded()) + retryCountOnTaskFailure: 10 # for any files being locked continueOnError: true - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - output: pipelineArtifact @@ -40,18 +38,18 @@ jobs: displayName: 'Publish logs' continueOnError: true condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts - ${{ if eq(parameters.enablePublishBuildArtifacts, true) }}: - - output: buildArtifacts + - output: pipelineArtifact displayName: Publish Logs - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' - publishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }} + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' + artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }} continueOnError: true condition: always() - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - output: pipelineArtifact @@ -59,14 +57,20 @@ jobs: artifactName: 'BuildConfiguration' displayName: 'Publish build retry configuration' continueOnError: true - sbomEnabled: false # we don't need SBOM for BuildConfiguration + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # BuildConfiguration is a non-production artifact - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + # V4 publishing: automatically publish staged artifacts as a pipeline artifact. + # The artifact name matches the SDK's FutureArtifactName ($(System.PhaseName)_Artifacts), + # which is encoded in the asset manifest for downstream publishing to discover. + # Jobs can opt in by setting enablePublishing: true. + - ${{ if and(eq(parameters.publishingVersion, 4), eq(parameters.enablePublishing, 'true')) }}: - output: pipelineArtifact - displayName: Publish SBOM manifest + displayName: 'Publish V4 pipeline artifacts' + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: '$(System.PhaseName)_Artifacts' continueOnError: true - targetPath: $(Build.ArtifactStagingDirectory)/sbom - artifactName: $(ARTIFACT_NAME) + retryCountOnTaskFailure: 10 # for any files being locked # add any outputs provided via root yaml - ${{ if ne(parameters.templateContext.outputs, '') }}: diff --git a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml index 172f9f0fdc97..9e5981365e56 100644 --- a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml +++ b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml @@ -24,5 +24,7 @@ steps: artifactName: ${{ parameters.args.artifactName }} ${{ if parameters.args.properties }}: properties: ${{ parameters.args.properties }} - ${{ if parameters.args.sbomEnabled }}: + ${{ if ne(parameters.args.sbomEnabled, '') }}: sbomEnabled: ${{ parameters.args.sbomEnabled }} + ${{ if ne(parameters.args.isProduction, '') }}: + isProduction: ${{ parameters.args.isProduction }} diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml index 1f308b24efc4..2cc3ae305d5a 100644 --- a/eng/common/templates-official/variables/pool-providers.yml +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -23,7 +23,7 @@ # # pool: # name: $(DncEngInternalBuildPool) -# image: 1es-windows-2022 +# image: windows.vs2026.amd64 variables: # Coalesce the target and source branches so we know when a PR targets a release branch diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 238fa0818f7b..5e261f34db42 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -1,12 +1,12 @@ parameters: enablePublishBuildArtifacts: false - disableComponentGovernance: '' - componentGovernanceIgnoreDirectories: '' -# Sbom related params - enableSbom: true runAsPublic: false - PackageVersion: 9.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' +# CG related params, unused now and can eventually be removed + disableComponentGovernance: unused +# Sbom related params, unused now and can eventually be removed + enableSbom: unused + PackageVersion: unused + BuildDropPath: unused jobs: - template: /eng/common/core-templates/job/job.yml @@ -21,32 +21,34 @@ jobs: - ${{ each step in parameters.steps }}: - ${{ step }} - componentGovernanceSteps: - - template: /eng/common/templates/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false - ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + # we don't run CG in public + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable artifactPublishSteps: - ${{ if ne(parameters.artifacts.publish, '') }}: - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: false args: displayName: Publish pipeline artifacts - pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - publishLocation: Container + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} continueOnError: true - condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked + condition: succeeded() + retryCountOnTaskFailure: 10 # for any files being locked + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: false + args: + displayName: Publish pipeline artifacts + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: not(succeeded()) + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: @@ -57,20 +59,19 @@ jobs: displayName: 'Publish logs' continueOnError: true condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: false args: displayName: Publish Logs - pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' - publishLocation: Container + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }} continueOnError: true condition: always() + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml @@ -81,4 +82,4 @@ jobs: artifactName: 'BuildConfiguration' displayName: 'Publish build retry configuration' continueOnError: true - sbomEnabled: false # we don't need SBOM for BuildConfiguration + retryCountOnTaskFailure: 10 # for any files being locked diff --git a/eng/common/templates/steps/vmr-sync.yml b/eng/common/templates/steps/vmr-sync.yml index 599afb6186b8..eb619c502683 100644 --- a/eng/common/templates/steps/vmr-sync.yml +++ b/eng/common/templates/steps/vmr-sync.yml @@ -38,27 +38,6 @@ steps: displayName: Label PR commit workingDirectory: $(Agent.BuildDirectory)/repo -- script: | - vmr_sha=$(grep -oP '(?<=Sha=")[^"]*' $(Agent.BuildDirectory)/repo/eng/Version.Details.xml) - echo "##vso[task.setvariable variable=vmr_sha]$vmr_sha" - displayName: Obtain the vmr sha from Version.Details.xml (Unix) - condition: ne(variables['Agent.OS'], 'Windows_NT') - workingDirectory: $(Agent.BuildDirectory)/repo - -- powershell: | - [xml]$xml = Get-Content -Path $(Agent.BuildDirectory)/repo/eng/Version.Details.xml - $vmr_sha = $xml.SelectSingleNode("//Source").Sha - Write-Output "##vso[task.setvariable variable=vmr_sha]$vmr_sha" - displayName: Obtain the vmr sha from Version.Details.xml (Windows) - condition: eq(variables['Agent.OS'], 'Windows_NT') - workingDirectory: $(Agent.BuildDirectory)/repo - -- script: | - git fetch --all - git checkout $(vmr_sha) - displayName: Checkout VMR at correct sha for repo flow - workingDirectory: ${{ parameters.vmrPath }} - - script: | git config --global user.name "dotnet-maestro[bot]" git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com" diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index e0b19c14a073..587770f0add4 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -23,7 +23,7 @@ # # pool: # name: $(DncEngInternalBuildPool) -# demands: ImageOverride -equals windows.vs2019.amd64 +# demands: ImageOverride -equals windows.vs2026.amd64 variables: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - template: /eng/common/templates-official/variables/pool-providers.yml diff --git a/eng/common/templates/vmr-build-pr.yml b/eng/common/templates/vmr-build-pr.yml index ce3c29a62faf..2f3694fa1323 100644 --- a/eng/common/templates/vmr-build-pr.yml +++ b/eng/common/templates/vmr-build-pr.yml @@ -34,6 +34,7 @@ resources: type: github name: dotnet/dotnet endpoint: dotnet + ref: refs/heads/main # Set to whatever VMR branch the PR build should insert into stages: - template: /eng/pipelines/templates/stages/vmr-build.yml@vmr diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 578705ee4dbd..65adefc7f268 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -34,6 +34,9 @@ # Configures warning treatment in msbuild. [bool]$warnAsError = if (Test-Path variable:warnAsError) { $warnAsError } else { $true } +# Specifies semi-colon delimited list of warning codes that should not be treated as errors. +[string]$warnNotAsError = if (Test-Path variable:warnNotAsError) { $warnNotAsError } else { '' } + # Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json). [string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null } @@ -157,9 +160,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { return $global:_DotNetInstallDir } - # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism - $env:DOTNET_MULTILEVEL_LOOKUP=0 - # Disable first run since we do not need all ASP.NET packages restored. $env:DOTNET_NOLOGO=1 @@ -185,7 +185,11 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { - $dotnetRoot = Join-Path $RepoRoot '.dotnet' + if (-not [string]::IsNullOrEmpty($env:DOTNET_GLOBAL_INSTALL_DIR)) { + $dotnetRoot = $env:DOTNET_GLOBAL_INSTALL_DIR + } else { + $dotnetRoot = Join-Path $RepoRoot '.dotnet' + } if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) { if ($install) { @@ -225,7 +229,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build Write-PipelinePrependPath -Path $dotnetRoot - Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' Write-PipelineSetVariable -Name 'DOTNET_NOLOGO' -Value '1' return $global:_DotNetInstallDir = $dotnetRoot @@ -277,7 +280,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) { Retry({ Write-Host "GET $uri" - Invoke-WebRequest $uri -OutFile $installScript + Invoke-WebRequest $uri -UseBasicParsing -OutFile $installScript }) } @@ -299,6 +302,8 @@ function InstallDotNet([string] $dotnetRoot, $dotnetVersionLabel = "'sdk v$version'" + # For performance this check is duplicated in src/Microsoft.DotNet.Arcade.Sdk/src/InstallDotNetCore.cs + # if you are making changes here, consider if you need to make changes there as well. if ($runtime -ne '' -and $runtime -ne 'sdk') { $runtimePath = $dotnetRoot $runtimePath = $runtimePath + "\shared" @@ -374,12 +379,11 @@ function InstallDotNet([string] $dotnetRoot, # # 1. MSBuild from an active VS command prompt # 2. MSBuild from a compatible VS installation -# 3. MSBuild from the xcopy tool package # # Returns full path to msbuild.exe. # Throws on failure. # -function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) { +function InitializeVisualStudioMSBuild([object]$vsRequirements = $null) { if (-not (IsWindowsPlatform)) { throw "Cannot initialize Visual Studio on non-Windows" } @@ -389,13 +393,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } # Minimum VS version to require. - $vsMinVersionReqdStr = '17.7' - $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) - - # If the version of msbuild is going to be xcopied, - # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/18.0.0 - $defaultXCopyMSBuildVersion = '18.0.0' + $vsMinVersionReqdStr = '18.0' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { @@ -425,46 +423,16 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } } - # Locate Visual Studio installation or download x-copy msbuild. + # Locate Visual Studio installation. $vsInfo = LocateVisualStudio $vsRequirements - if ($vsInfo -ne $null -and $env:ForceUseXCopyMSBuild -eq $null) { + if ($vsInfo -ne $null) { # Ensure vsInstallDir has a trailing slash $vsInstallDir = Join-Path $vsInfo.installationPath "\" $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion } else { - if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') { - $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' - $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] - } else { - #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download - if($vsMinVersion -lt $vsMinVersionReqd){ - Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible" - $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion - $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] - } - else{ - # If the VS version IS compatible, look for an xcopy msbuild package - # with a version matching VS. - # Note: If this version does not exist, then an explicit version of xcopy msbuild - # can be specified in global.json. This will be required for pre-release versions of msbuild. - $vsMajorVersion = $vsMinVersion.Major - $vsMinorVersion = $vsMinVersion.Minor - $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0" - } - } - - $vsInstallDir = $null - if ($xcopyMSBuildVersion.Trim() -ine "none") { - $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install - if ($vsInstallDir -eq $null) { - throw "Could not xcopy msbuild. Please check that package 'Microsoft.DotNet.Arcade.MSBuild.Xcopy @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." - } - } - if ($vsInstallDir -eq $null) { - throw 'Unable to find Visual Studio that has required version and components installed' - } + throw 'Unable to find Visual Studio that has required version and components installed' } $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } @@ -491,38 +459,6 @@ function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [str } } -function InstallXCopyMSBuild([string]$packageVersion) { - return InitializeXCopyMSBuild $packageVersion -install $true -} - -function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { - $packageName = 'Microsoft.DotNet.Arcade.MSBuild.Xcopy' - $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" - $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" - - if (!(Test-Path $packageDir)) { - if (!$install) { - return $null - } - - Create-Directory $packageDir - - Write-Host "Downloading $packageName $packageVersion" - $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - Retry({ - Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath - }) - - if (!(Test-Path $packagePath)) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "See https://dev.azure.com/dnceng/internal/_wiki/wikis/DNCEng%20Services%20Wiki/1074/Updating-Microsoft.DotNet.Arcade.MSBuild.Xcopy-WAS-RoslynTools.MSBuild-(xcopy-msbuild)-generation?anchor=troubleshooting for help troubleshooting issues with XCopy MSBuild" - throw - } - Unzip $packagePath $packageDir - } - - return Join-Path $packageDir 'tools' -} - # # Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json. # @@ -544,7 +480,6 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') { $vswhereVersion = $GlobalJson.tools.vswhere } else { - # keep this in sync with the VSWhereVersion in DefaultVersions.props $vswhereVersion = '3.1.7' } @@ -556,23 +491,30 @@ function LocateVisualStudio([object]$vsRequirements = $null){ Write-Host "Downloading vswhere $vswhereVersion" $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Retry({ - Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe + Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -UseBasicParsing -OutFile $vswhereExe }) } - if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } + if (!$vsRequirements) { + if (Get-Member -InputObject $GlobalJson.tools -Name 'vs' -ErrorAction SilentlyContinue) { + $vsRequirements = $GlobalJson.tools.vs + } else { + $vsRequirements = $null + } + } + $args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*') if (!$excludePrereleaseVS) { $args += '-prerelease' } - if (Get-Member -InputObject $vsRequirements -Name 'version') { + if ($vsRequirements -and (Get-Member -InputObject $vsRequirements -Name 'version' -ErrorAction SilentlyContinue)) { $args += '-version' $args += $vsRequirements.version } - if (Get-Member -InputObject $vsRequirements -Name 'components') { + if ($vsRequirements -and (Get-Member -InputObject $vsRequirements -Name 'components' -ErrorAction SilentlyContinue)) { foreach ($component in $vsRequirements.components) { $args += '-requires' $args += $component @@ -585,6 +527,11 @@ function LocateVisualStudio([object]$vsRequirements = $null){ return $null } + if ($null -eq $vsInfo -or $vsInfo.Count -eq 0) { + throw "No instance of Visual Studio meeting the requirements specified was found. Requirements: $($args -join ' ')" + return $null + } + # use first matching instance return $vsInfo[0] } @@ -620,7 +567,7 @@ function InitializeBuildTool() { $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net' } } elseif ($msbuildEngine -eq "vs") { try { - $msbuildPath = InitializeVisualStudioMSBuild -install:$restore + $msbuildPath = InitializeVisualStudioMSBuild } catch { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ ExitWithExitCode 1 @@ -667,7 +614,17 @@ function GetNuGetPackageCachePath() { # Returns a full path to an Arcade SDK task project file. function GetSdkTaskProject([string]$taskName) { - return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj" + $toolsetDir = Split-Path (InitializeToolset) -Parent + $proj = Join-Path $toolsetDir "$taskName.proj" + if (Test-Path $proj) { + return $proj + } + # TODO: Remove this fallback once all supported versions use the new layout. + $legacyProj = Join-Path $toolsetDir "SdkTasks\$taskName.proj" + if (Test-Path $legacyProj) { + return $legacyProj + } + throw "Unable to find $taskName.proj in toolset at: $toolsetDir" } function InitializeNativeTools() { @@ -704,13 +661,18 @@ function InitializeToolset() { $nugetCache = GetNuGetPackageCachePath $toolsetVersion = Read-ArcadeSdkVersion - $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt" + $toolsetToolsDir = Join-Path $ToolsetDir $toolsetVersion - if (Test-Path $toolsetLocationFile) { - $path = Get-Content $toolsetLocationFile -TotalCount 1 - if (Test-Path $path) { - return $global:_InitializeToolset = $path - } + # Check if the toolset has already been extracted + $toolsetBuildProj = $null + $buildProjPath = Join-Path $toolsetToolsDir 'Build.proj' + + if (Test-Path $buildProjPath) { + $toolsetBuildProj = $buildProjPath + } + + if ($toolsetBuildProj -ne $null) { + return $global:_InitializeToolset = $toolsetBuildProj } if (-not $restore) { @@ -718,21 +680,50 @@ function InitializeToolset() { ExitWithExitCode 1 } - $buildTool = InitializeBuildTool + $downloadArgs = @("package", "download", "Microsoft.DotNet.Arcade.Sdk@$toolsetVersion", "--verbosity", "minimal", "--prerelease", "--output", "$nugetCache") + $nugetConfig = $env:NUGET_CONFIG + if (-not $nugetConfig) { + # Search for any variation of nuget.config in the RepoRoot + $configFile = Get-ChildItem -Path $RepoRoot -File | Where-Object { $_.Name -ieq "nuget.config" } | Select-Object -First 1 + + if ($configFile) { + $nugetConfig = $configFile.FullName + } + } - $proj = Join-Path $ToolsetDir 'restore.proj' - $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'ToolsetRestore.binlog') } else { '' } + if ($nugetConfig) { + $downloadArgs += "--configfile" + $downloadArgs += $nugetConfig + } + DotNet @downloadArgs + + $packageDir = Join-Path $nugetCache (Join-Path 'microsoft.dotnet.arcade.sdk' $toolsetVersion) + $packageToolsetDir = Join-Path $packageDir 'toolset' + $packageToolsDir = Join-Path $packageDir 'tools' - '' | Set-Content $proj + # TODO: Remove the tools/ check once all supported versions have the toolset folder. + if (!(Test-Path $packageToolsetDir) -and !(Test-Path $packageToolsDir)) { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Arcade SDK package does not contain a toolset or tools folder: $packageDir" + ExitWithExitCode 3 + } - MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile + New-Item -ItemType Directory -Path $toolsetToolsDir -Force | Out-Null + + # Copy toolset if present at the package root (new layout), otherwise fall back to tools + if (Test-Path $packageToolsetDir) { + Copy-Item -Path "$packageToolsetDir\*" -Destination $toolsetToolsDir -Recurse -Force + } else { + # TODO: Remove this fallback once all supported versions have the toolset folder. + Copy-Item -Path "$packageToolsDir\*" -Destination $toolsetToolsDir -Recurse -Force + } - $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1 - if (!(Test-Path $path)) { - throw "Invalid toolset path: $path" + if (Test-Path $buildProjPath) { + $toolsetBuildProj = $buildProjPath + } else { + throw "Unable to find Build.proj in toolset at: $toolsetToolsDir" } - return $global:_InitializeToolset = $path + return $global:_InitializeToolset = $toolsetBuildProj } function ExitWithExitCode([int] $exitCode) { @@ -793,6 +784,40 @@ function MSBuild() { MSBuild-Core @args } +# +# Executes a dotnet command with arguments passed to the function. +# Terminates the script if the command fails. +# +function DotNet() { + $dotnetRoot = InitializeDotNetCli -install:$restore + $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') + + $cmdArgs = "" + foreach ($arg in $args) { + if ($null -ne $arg -and $arg.Trim() -ne "") { + if ($arg.EndsWith('\')) { + $arg = $arg + "\" + } + $cmdArgs += " `"$arg`"" + } + } + + $env:ARCADE_BUILD_TOOL_COMMAND = "`"$dotnetPath`" $cmdArgs" + + $exitCode = Exec-Process $dotnetPath $cmdArgs + + if ($exitCode -ne 0) { + Write-Host "dotnet command failed with exit code $exitCode. Check errors above." -ForegroundColor Red + + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR) { + Write-PipelineSetResult -Result "Failed" -Message "dotnet command execution failed." + ExitWithExitCode 0 + } else { + ExitWithExitCode $exitCode + } + } +} + # # Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. # The arguments are automatically quoted. @@ -817,6 +842,11 @@ function MSBuild-Core() { $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" + # Add -mt flag for MSBuild multithreaded mode if enabled via environment variable + if ($env:MSBUILD_MT_ENABLED -eq "1") { + $cmdArgs += ' -mt' + } + if ($warnAsError) { $cmdArgs += ' /warnaserror /p:TreatWarningsAsErrors=true' } @@ -824,6 +854,10 @@ function MSBuild-Core() { $cmdArgs += ' /p:TreatWarningsAsErrors=false' } + if ($warnAsError -and $warnNotAsError) { + $cmdArgs += " /warnnotaserror:$warnNotAsError /p:AdditionalWarningsNotAsErrors=$warnNotAsError" + } + foreach ($arg in $args) { if ($null -ne $arg -and $arg.Trim() -ne "") { if ($arg.EndsWith('\')) { diff --git a/eng/common/tools.sh b/eng/common/tools.sh index c1841c9dfd0f..95c55ce9b4d9 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -52,6 +52,9 @@ fi # Configures warning treatment in msbuild. warn_as_error=${warn_as_error:-true} +# Specifies semi-colon delimited list of warning codes that should not be treated as errors. +warn_not_as_error=${warn_not_as_error:-''} + # True to attempt using .NET Core already that meets requirements specified in global.json # installed on the machine instead of downloading one. use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} @@ -115,9 +118,6 @@ function InitializeDotNetCli { local install=$1 - # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism - export DOTNET_MULTILEVEL_LOOKUP=0 - # Disable first run since we want to control all package sources export DOTNET_NOLOGO=1 @@ -148,7 +148,11 @@ function InitializeDotNetCli { if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else - dotnet_root="${repo_root}.dotnet" + if [[ -n "${DOTNET_GLOBAL_INSTALL_DIR:-}" ]]; then + dotnet_root="$DOTNET_GLOBAL_INSTALL_DIR" + else + dotnet_root="${repo_root}.dotnet" + fi export DOTNET_INSTALL_DIR="$dotnet_root" @@ -166,7 +170,6 @@ function InitializeDotNetCli { # build steps from using anything other than what we've downloaded. Write-PipelinePrependPath -path "$dotnet_root" - Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0" Write-PipelineSetVariable -name "DOTNET_NOLOGO" -value "1" # return value @@ -188,6 +191,8 @@ function InstallDotNet { local version=$2 local runtime=$4 + # For performance this check is duplicated in src/Microsoft.DotNet.Arcade.Sdk/src/InstallDotNetCore.cs + # if you are making changes here, consider if you need to make changes there as well. local dotnetVersionLabel="'$runtime v$version'" if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then runtimePath="$root" @@ -406,15 +411,18 @@ function InitializeToolset { ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" local toolset_version=$_ReadGlobalVersion - local toolset_location_file="$toolset_dir/$toolset_version.txt" + local toolset_tools_dir="$toolset_dir/$toolset_version" - if [[ -a "$toolset_location_file" ]]; then - local path=`cat "$toolset_location_file"` - if [[ -a "$path" ]]; then - # return value - _InitializeToolset="$path" - return - fi + # Check if the toolset has already been extracted + local toolset_build_proj="" + if [[ -a "$toolset_tools_dir/Build.proj" ]]; then + toolset_build_proj="$toolset_tools_dir/Build.proj" + fi + + if [[ -n "$toolset_build_proj" ]]; then + # return value + _InitializeToolset="$toolset_build_proj" + return fi if [[ "$restore" != true ]]; then @@ -422,20 +430,45 @@ function InitializeToolset { ExitWithExitCode 2 fi - local proj="$toolset_dir/restore.proj" + local download_args=("package" "download" "Microsoft.DotNet.Arcade.Sdk@$toolset_version" "--verbosity" "minimal" "--prerelease" "--output" "$_GetNuGetPackageCachePath") + local nuget_config="${NUGET_CONFIG:-}" + if [[ -z "$nuget_config" ]]; then + # Search for any variation of nuget.config in the RepoRoot + local found_config + found_config=$(find "$repo_root" -maxdepth 1 -type f -iname "nuget.config" -print -quit) - local bl="" - if [[ "$binary_log" == true ]]; then - bl="/bl:$log_dir/ToolsetRestore.binlog" + if [[ -n "$found_config" ]]; then + nuget_config="$found_config" + fi fi - echo '' > "$proj" - MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" + if [[ -n "$nuget_config" ]]; then + download_args+=("--configfile" "$nuget_config") + fi + DotNet "${download_args[@]}" + + local package_dir="$_GetNuGetPackageCachePath/microsoft.dotnet.arcade.sdk/$toolset_version" + + # TODO: Remove the tools/ check once all supported versions have the toolset folder. + if [[ ! -d "$package_dir/toolset" && ! -d "$package_dir/tools" ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Arcade SDK package does not contain a toolset or tools folder: $package_dir" + ExitWithExitCode 3 + fi - local toolset_build_proj=`cat "$toolset_location_file"` + mkdir -p "$toolset_tools_dir" - if [[ ! -a "$toolset_build_proj" ]]; then - Write-PipelineTelemetryError -category 'Build' "Invalid toolset path: $toolset_build_proj" + # Copy toolset if present at the package root (new layout), otherwise fall back to tools + if [[ -d "$package_dir/toolset" ]]; then + cp -r "$package_dir/toolset/." "$toolset_tools_dir" + else + # TODO: Remove this fallback once all supported versions have the toolset folder. + cp -r "$package_dir/tools/." "$toolset_tools_dir" + fi + + if [[ -a "$toolset_tools_dir/Build.proj" ]]; then + toolset_build_proj="$toolset_tools_dir/Build.proj" + else + Write-PipelineTelemetryError -category 'Build' "Unable to find Build.proj in toolset at: $toolset_tools_dir" ExitWithExitCode 3 fi @@ -457,6 +490,26 @@ function StopProcesses { return 0 } +function DotNet { + InitializeDotNetCli $restore + + local dotnet_path="$_InitializeDotNetCli/dotnet" + + export ARCADE_BUILD_TOOL_COMMAND="$dotnet_path $@" + + "$dotnet_path" "$@" || { + local exit_code=$? + echo "dotnet command failed with exit code $exit_code. Check errors above." + + if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$from_vmr" != true ]]; then + Write-PipelineSetResult -result "Failed" -message "dotnet command execution failed." + ExitWithExitCode 0 + else + ExitWithExitCode $exit_code + fi + } +} + function MSBuild { local args=( "$@" ) if [[ "$pipelines_log" == true ]]; then @@ -526,7 +579,18 @@ function MSBuild-Core { } } - RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" + # Add -mt flag for MSBuild multithreaded mode if enabled via environment variable + local mt_switch="" + if [[ "${MSBUILD_MT_ENABLED:-}" == "1" ]]; then + mt_switch="-mt" + fi + + local warnnotaserror_switch="" + if [[ -n "$warn_not_as_error" && "$warn_as_error" == true ]]; then + warnnotaserror_switch="/warnnotaserror:$warn_not_as_error /p:AdditionalWarningsNotAsErrors=$warn_not_as_error" + fi + + RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch $warnnotaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } function GetDarc { @@ -543,8 +607,22 @@ function GetDarc { # Returns a full path to an Arcade SDK task project file. function GetSdkTaskProject { - taskName=$1 - echo "$(dirname $_InitializeToolset)/SdkTasks/$taskName.proj" + local taskName=$1 + local toolsetDir + toolsetDir="$(dirname "$_InitializeToolset")" + local proj="$toolsetDir/$taskName.proj" + if [[ -a "$proj" ]]; then + echo "$proj" + return + fi + # TODO: Remove this fallback once all supported versions use the new layout. + local legacyProj="$toolsetDir/SdkTasks/$taskName.proj" + if [[ -a "$legacyProj" ]]; then + echo "$legacyProj" + return + fi + Write-PipelineTelemetryError -category 'Build' "Unable to find $taskName.proj in toolset at: $toolsetDir" + ExitWithExitCode 3 } ResolvePath "${BASH_SOURCE[0]}" diff --git a/eng/common/vmr-sync.ps1 b/eng/common/vmr-sync.ps1 index 97302f3205be..b37992d91cf0 100644 --- a/eng/common/vmr-sync.ps1 +++ b/eng/common/vmr-sync.ps1 @@ -103,12 +103,20 @@ Set-StrictMode -Version Latest Highlight 'Installing .NET, preparing the tooling..' . .\eng\common\tools.ps1 $dotnetRoot = InitializeDotNetCli -install:$true +$env:DOTNET_ROOT = $dotnetRoot $darc = Get-Darc -$dotnet = "$dotnetRoot\dotnet.exe" Highlight "Starting the synchronization of VMR.." # Synchronize the VMR +$versionDetailsPath = Resolve-Path (Join-Path $PSScriptRoot '..\Version.Details.xml') | Select-Object -ExpandProperty Path +[xml]$versionDetails = Get-Content -Path $versionDetailsPath +$repoName = $versionDetails.SelectSingleNode('//Source').Mapping +if (-not $repoName) { + Fail "Failed to resolve repo mapping from $versionDetailsPath" + exit 1 +} + $darcArgs = ( "vmr", "forwardflow", "--tmp", $tmpDir, @@ -130,9 +138,27 @@ if ($LASTEXITCODE -eq 0) { Highlight "Synchronization succeeded" } else { - Fail "Synchronization of repo to VMR failed!" - Fail "'$vmrDir' is left in its last state (re-run of this script will reset it)." - Fail "Please inspect the logs which contain path to the failing patch file (use -debugOutput to get all the details)." - Fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script." - exit 1 + Highlight "Failed to flow code into the local VMR. Falling back to resetting the VMR to match repo contents..." + git -C $vmrDir reset --hard + + $resetArgs = ( + "vmr", "reset", + "${repoName}:HEAD", + "--vmr", $vmrDir, + "--tmp", $tmpDir, + "--additional-remotes", "${repoName}:${repoRoot}" + ) + + & "$darc" $resetArgs + + if ($LASTEXITCODE -eq 0) { + Highlight "Successfully reset the VMR using 'darc vmr reset'" + } + else { + Fail "Synchronization of repo to VMR failed!" + Fail "'$vmrDir' is left in its last state (re-run of this script will reset it)." + Fail "Please inspect the logs which contain path to the failing patch file (use -debugOutput to get all the details)." + Fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script." + exit 1 + } } diff --git a/eng/common/vmr-sync.sh b/eng/common/vmr-sync.sh index 44239e331c0c..198caec59bd4 100644 --- a/eng/common/vmr-sync.sh +++ b/eng/common/vmr-sync.sh @@ -186,6 +186,13 @@ fi # Synchronize the VMR +version_details_path=$(cd "$scriptroot/.."; pwd -P)/Version.Details.xml +repo_name=$(grep -m 1 ' GetDataAsync(IReadOnlyDictionary query } }); - [Theory] #if !ANDROID // Custom schemes are not supported on Android #if !WINDOWS // TODO: There seems to be a bug with the implementation in the WASDK version of WebView2 + [Theory] [InlineData("app://echoservice/")] #endif #endif #if !IOS && !MACCATALYST // Cannot intercept https requests on iOS/MacCatalyst + [Theory(Skip = "Flaky due to external service dependency (echo.free.beeceptor.com). See https://github.com/dotnet/maui/issues/33927")] [InlineData("https://echo.free.beeceptor.com/sample-request")] #endif public Task RequestsCanBeInterceptedAndCustomDataReturnedForDifferentHosts(string uriBase) => @@ -287,13 +288,14 @@ public Task RequestsCanBeInterceptedAndHeadersAddedForDifferentHosts(string uriB Assert.Equal(ExpectedHeaderValue, actualHeaderValue); }); - [Theory] #if !ANDROID // Custom schemes are not supported on Android #if !WINDOWS // TODO: There seems to be a bug with the implementation in the WASDK version of WebView2 + [Theory] [InlineData("app://echoservice/")] #endif #endif #if !IOS && !MACCATALYST // Cannot intercept https requests on iOS/MacCatalyst + [Theory(Skip = "Flaky due to external service dependency (echo.free.beeceptor.com). See https://github.com/dotnet/maui/issues/33927")] [InlineData("https://echo.free.beeceptor.com/sample-request")] #endif public Task RequestsCanBeInterceptedAndCancelledForDifferentHosts(string uriBase) => @@ -330,13 +332,14 @@ public Task RequestsCanBeInterceptedAndCancelledForDifferentHosts(string uriBase Assert.True(result); }); - [Theory] #if !ANDROID // Custom schemes are not supported on Android #if !WINDOWS // TODO: There seems to be a bug with the implementation in the WASDK version of WebView2 + [Theory] [InlineData("app://echoservice/")] #endif #endif #if !IOS && !MACCATALYST // Cannot intercept https requests on iOS/MacCatalyst + [Theory(Skip = "Flaky due to external service dependency (echo.free.beeceptor.com). See https://github.com/dotnet/maui/issues/33927")] [InlineData("https://echo.free.beeceptor.com/sample-request")] #endif public Task RequestsCanBeInterceptedAndCaseInsensitiveHeadersRead(string uriBase) => diff --git a/src/Controls/tests/DeviceTests/Elements/HybridWebView/HybridWebViewTests_Interception.cs b/src/Controls/tests/DeviceTests/Elements/HybridWebView/HybridWebViewTests_Interception.cs index a2cbc177d37d..65bdf373e34e 100644 --- a/src/Controls/tests/DeviceTests/Elements/HybridWebView/HybridWebViewTests_Interception.cs +++ b/src/Controls/tests/DeviceTests/Elements/HybridWebView/HybridWebViewTests_Interception.cs @@ -280,13 +280,14 @@ public Task RequestsCanBeInterceptedAndHeadersAddedForDifferentHosts(string uriB Assert.Equal(ExpectedHeaderValue, actualHeaderValue); }); - [Theory] #if !ANDROID // Custom schemes are not supported on Android #if !WINDOWS // TODO: There seems to be a bug with the implementation in the WASDK version of WebView2 + [Theory] [InlineData("app://echoservice/", "RequestsWithCustomSchemeCanBeIntercepted")] #endif #endif #if !IOS && !MACCATALYST // Cannot intercept https requests on iOS/MacCatalyst + [Theory(Skip = "Flaky due to external service dependency (echo.free.beeceptor.com). See https://github.com/dotnet/maui/issues/33927")] [InlineData("https://echo.free.beeceptor.com/", "RequestsCanBeIntercepted")] #endif public Task RequestsCanBeInterceptedAndCancelledForDifferentHosts(string uriBase, string function) =>