diff --git a/.github/scripts/Aggregate-CopilotTokenUsage.Tests.ps1 b/.github/scripts/Aggregate-CopilotTokenUsage.Tests.ps1 new file mode 100644 index 000000000000..dadd038730b3 --- /dev/null +++ b/.github/scripts/Aggregate-CopilotTokenUsage.Tests.ps1 @@ -0,0 +1,95 @@ +#!/usr/bin/env pwsh +#Requires -Modules Pester +<# +.SYNOPSIS + Pester tests for Aggregate-CopilotTokenUsage.ps1. +#> + +Describe 'Aggregate-CopilotTokenUsage.ps1' { + BeforeEach { + $script:fixtureRoot = Join-Path ([System.IO.Path]::GetTempPath()) "token-usage-fixtures-$([guid]::NewGuid())" + $script:inputRoot = Join-Path $script:fixtureRoot 'input' + $script:outputRoot = Join-Path $script:fixtureRoot 'output' + New-Item -ItemType Directory -Path $script:inputRoot -Force | Out-Null + } + + AfterEach { + Remove-Item -Path $script:fixtureRoot -Recurse -Force -ErrorAction SilentlyContinue + } + + It 'writes raw and summarized artifacts with zero rows for stages without Copilot invocations' { + $nested = Join-Path $script:inputRoot 'CopilotLogs/copilot-token-usage/raw' + New-Item -ItemType Directory -Path $nested -Force | Out-Null + + [ordered]@{ + schemaVersion = 1 + prNumber = 35677 + pipeline = [ordered]@{ stageName = 'ReviewPR' } + scriptPhase = 'CopilotReview' + copilotStep = 'STEP 5a: TRY-FIX' + model = 'gpt-5.5' + durationMs = 5000 + apiDurationMs = 2000 + turnCount = 2 + toolCount = 3 + cliUsage = [ordered]@{ + aicUsed = 7.5 + contextWindow = 1100000 + contextWindowRaw = '1.1M' + } + normalizedTokens = [ordered]@{ + inputTokens = 100 + outputTokens = 40 + cachedInputTokens = 10 + reasoningOutputTokens = 5 + totalTokens = 140 + } + } | ConvertTo-Json -Depth 10 | Set-Content (Join-Path $nested 'copilot-token-usage-a.json') -Encoding UTF8 + + $scriptPath = Join-Path $PSScriptRoot 'shared/Aggregate-CopilotTokenUsage.ps1' + & $scriptPath ` + -InputRoot $script:inputRoot ` + -OutputDir $script:outputRoot ` + -PRNumber '35677' ` + -ExpectedStages @('ReviewPR', 'RunDeepUITests', 'UpdateAISummaryComment', 'AnalyzeCopilotTokenUsage') + + Test-Path (Join-Path $script:outputRoot 'token-usage-raw.jsonl') | Should -Be $true + Test-Path (Join-Path $script:outputRoot 'token-usage-summary.md') | Should -Be $true + Test-Path (Join-Path $script:outputRoot 'token-usage-by-step.csv') | Should -Be $true + + $summary = Get-Content (Join-Path $script:outputRoot 'token-usage-summary.json') -Raw | ConvertFrom-Json + $summary.recordCount | Should -Be 1 + $summary.totals.inputTokens | Should -Be 100 + $summary.totals.outputTokens | Should -Be 40 + $summary.totals.cachedInputTokens | Should -Be 10 + $summary.totals.reasoningOutputTokens | Should -Be 5 + $summary.totals.totalTokens | Should -Be 140 + $summary.totals.aicUsed | Should -Be 7.5 + + $reviewStage = $summary.stages | Where-Object { $_.stageName -eq 'ReviewPR' } + $reviewStage.invocationCount | Should -Be 1 + $reviewStage.totalTokens | Should -Be 140 + $reviewStage.reasoningOutputTokens | Should -Be 5 + $reviewStage.aicUsed | Should -Be 7.5 + + $deepStage = $summary.stages | Where-Object { $_.stageName -eq 'RunDeepUITests' } + $deepStage.invocationCount | Should -Be 0 + $deepStage.totalTokens | Should -Be 0 + $deepStage.reasoningOutputTokens | Should -Be 0 + $deepStage.aicUsed | Should -Be 0 + $deepStage.note | Should -Be 'No Copilot invocation observed in this stage.' + } + + It 'emits a no-record summary when the input artifact is missing' { + $scriptPath = Join-Path $PSScriptRoot 'shared/Aggregate-CopilotTokenUsage.ps1' + & $scriptPath ` + -InputRoot (Join-Path $script:fixtureRoot 'missing') ` + -OutputDir $script:outputRoot ` + -PRNumber '35677' + + $summary = Get-Content (Join-Path $script:outputRoot 'token-usage-summary.json') -Raw | ConvertFrom-Json + $summary.recordCount | Should -Be 0 + ($summary.stages | Where-Object { $_.stageName -eq 'ReviewPR' }).invocationCount | Should -Be 0 + Test-Path (Join-Path $script:outputRoot 'token-usage-by-step.csv') | Should -Be $true + } +} diff --git a/.github/scripts/Post-AISummaryComment.Tests.ps1 b/.github/scripts/Post-AISummaryComment.Tests.ps1 index ee0d3767ba62..f9c4e0994b37 100644 --- a/.github/scripts/Post-AISummaryComment.Tests.ps1 +++ b/.github/scripts/Post-AISummaryComment.Tests.ps1 @@ -20,6 +20,7 @@ BeforeAll { foreach ($functionName in @( 'Test-PhaseContentIsNoOp', 'Get-AIReviewEvent', + 'Test-RunValidationFailed', 'Test-HasNonPRWinner', 'Get-AIReviewEventForRun', 'New-FutureActionSection' @@ -132,6 +133,46 @@ Describe 'Get-AIReviewEventForRun' { Should -Be 'APPROVE' } + It 'vetoes APPROVE to REQUEST_CHANGES when the trusted gate-result is FAILED' { + $gateDir = Join-Path $script:testDir 'gate' + New-Item -ItemType Directory -Path $gateDir -Force | Out-Null + 'FAILED' | Set-Content (Join-Path $gateDir 'gate-result.txt') -Encoding UTF8 + + Get-AIReviewEventForRun -ReportContent '## ✅ Final Recommendation: APPROVE' -PRAgentDir $script:testDir | + Should -Be 'REQUEST_CHANGES' + } + + It 'keeps APPROVE when the trusted gate-result is PASSED (ignores a forged content.md)' { + $gateDir = Join-Path $script:testDir 'gate' + New-Item -ItemType Directory -Path $gateDir -Force | Out-Null + 'PASSED' | Set-Content (Join-Path $gateDir 'gate-result.txt') -Encoding UTF8 + # A forged content.md claiming PASSED must be irrelevant — the veto keys off gate-result.txt. + 'Gate Result: ✅ PASSED' | Set-Content (Join-Path $gateDir 'content.md') -Encoding UTF8 + + Get-AIReviewEventForRun -ReportContent '## ✅ Final Recommendation: APPROVE' -PRAgentDir $script:testDir | + Should -Be 'APPROVE' + } + + It 'vetoes APPROVE when deep UI tests report failures (real render format)' { + $uiDir = Join-Path $script:testDir 'uitests' + New-Item -ItemType Directory -Path $uiDir -Force | Out-Null + '❌ **Deep UI tests** — 12 passed, 3 failed across 4 categories on platform-pool agent (replaces in-process counts above).' | + Set-Content (Join-Path $uiDir 'content.md') -Encoding UTF8 + + Get-AIReviewEventForRun -ReportContent 'Final Recommendation: APPROVE' -PRAgentDir $script:testDir | + Should -Be 'REQUEST_CHANGES' + } + + It 'keeps APPROVE when deep UI tests pass (TRX-marked-failed wording does not false-trigger)' { + $uiDir = Join-Path $script:testDir 'uitests' + New-Item -ItemType Directory -Path $uiDir -Force | Out-Null + '✅ **Deep UI tests** — 50 passed; 2 setup categories (1 marked failed by TRX) across 4 categories on platform-pool agent.' | + Set-Content (Join-Path $uiDir 'content.md') -Encoding UTF8 + + Get-AIReviewEventForRun -ReportContent 'Final Recommendation: APPROVE' -PRAgentDir $script:testDir | + Should -Be 'APPROVE' + } + It 'does not force changes for missing, malformed, or PR-fix winner files' { Get-AIReviewEventForRun -ReportContent '' -PRAgentDir $script:testDir | Should -Be 'COMMENT' diff --git a/.github/scripts/Review-PR.Tests.ps1 b/.github/scripts/Review-PR.Tests.ps1 index 3d94ccc038ca..339faa65aa88 100644 --- a/.github/scripts/Review-PR.Tests.ps1 +++ b/.github/scripts/Review-PR.Tests.ps1 @@ -7,6 +7,7 @@ - Get-TrxResults (parses VSTest TRX produced by `dotnet test --logger trx`) - Get-DotNetTestResults (legacy console-output scraper, still used as fallback when TRX is missing) + - Copilot token usage helpers These functions sit on the critical path of STEP 3 (UI Test Execution Results in the AI summary review). A regression here can silently @@ -41,6 +42,206 @@ BeforeAll { Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'Get-TrxResults') Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'Get-DotNetTestResults') + Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'Test-IsNumericValue') + Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'ConvertTo-AzdoSafeConsole') + Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'Get-ObjectMemberValue') + Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'Get-CopilotUsageTokenFields') + Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'Get-TokenFieldSum') + Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'Get-TokenFieldPathDepth') + Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'Select-CanonicalTokenFields') + Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'Get-CopilotTokenMetrics') + Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'Convert-CopilotCompactNumber') + Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'Get-CopilotCliUsageLineData') + Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'Get-CopilotOtelTokenMetrics') + Invoke-Expression (Get-FunctionBody -ScriptText $content -FunctionName 'New-CopilotTokenUsageRecord') +} + +Describe 'Copilot token usage helpers' { + It 'normalizes known token fields while preserving raw token field paths' { + $usage = [pscustomobject]@{ + inputTokens = 100 + outputTokens = 40 + totalApiDurationMs = 1234 + nested = [pscustomobject]@{ + cachedInputTokens = 12 + } + } + + $metrics = Get-CopilotTokenMetrics -Usage $usage + + $metrics.inputTokens | Should -Be 100 + $metrics.outputTokens | Should -Be 40 + $metrics.cachedInputTokens | Should -Be 12 + $metrics.totalTokens | Should -Be 140 + @($metrics.rawTokenFields).Count | Should -Be 3 + @($metrics.rawTokenFields | Where-Object { $_.Path -eq 'nested.cachedInputTokens' }).Count | Should -Be 1 + } + + It 'prefers the root token aggregate over a nested per-model breakdown (no double-count)' { + # Regression guard: a payload carrying BOTH a root aggregate and a per-model + # breakdown must not sum both (1000 + 600 + 400 = 2000); the root wins. + $usage = [pscustomobject]@{ + inputTokens = 1000 + outputTokens = 200 + perModel = @( + [pscustomobject]@{ inputTokens = 600; outputTokens = 120 }, + [pscustomobject]@{ inputTokens = 400; outputTokens = 80 } + ) + } + + $metrics = Get-CopilotTokenMetrics -Usage $usage + + $metrics.inputTokens | Should -Be 1000 + $metrics.outputTokens | Should -Be 200 + } + + It 'sums a nested-only token breakdown when no root aggregate exists' { + # When only the per-model breakdown is present, it should be summed. + $usage = [pscustomobject]@{ + perModel = @( + [pscustomobject]@{ inputTokens = 600 }, + [pscustomobject]@{ inputTokens = 400 } + ) + } + + $metrics = Get-CopilotTokenMetrics -Usage $usage + + $metrics.inputTokens | Should -Be 1000 + } + + It 'parses Copilot CLI AIC and context footer lines' { + $aicLine = Get-CopilotCliUsageLineData -Line 'Session: 1030 AIC used' + $contextLine = Get-CopilotCliUsageLineData -Line 'GPT-5.5 • 1.1M context' + + $aicLine.aicUsed | Should -Be 1030 + $contextLine.model | Should -Be 'GPT-5.5' + $contextLine.contextWindowRaw | Should -Be '1.1M' + $contextLine.contextWindow | Should -Be 1100000 + } + + It 'reads token counts from Copilot OTel spans with both cache/reasoning naming variants' { + $otelPath = Join-Path ([System.IO.Path]::GetTempPath()) "copilot-otel-$([guid]::NewGuid()).jsonl" + try { + @( + [ordered]@{ + type = 'span' + attributes = [ordered]@{ + 'gen_ai.usage.input_tokens' = 1000 + 'gen_ai.usage.output_tokens' = 200 + 'gen_ai.usage.cache_read.input_tokens' = 800 + 'gen_ai.usage.reasoning.output_tokens' = 50 + 'github.copilot.cost' = 7.5 + } + }, + [ordered]@{ + type = 'span' + attributes = [ordered]@{ + 'gen_ai.usage.input_tokens' = 500 + 'gen_ai.usage.output_tokens' = 40 + 'gen_ai.usage.cache_read_input_tokens' = 400 + 'gen_ai.usage.reasoning_output_tokens' = 10 + } + } + ) | ForEach-Object { $_ | ConvertTo-Json -Depth 10 -Compress } | Set-Content $otelPath -Encoding UTF8 + + $metrics = Get-CopilotOtelTokenMetrics -Path $otelPath + + $metrics.available | Should -Be $true + $metrics.inputTokens | Should -Be 1500 + $metrics.outputTokens | Should -Be 240 + $metrics.cachedInputTokens | Should -Be 1200 + $metrics.reasoningOutputTokens | Should -Be 60 + $metrics.totalTokens | Should -Be 1740 + $metrics.copilotCost | Should -Be 7.5 + } finally { + Remove-Item $otelPath -Force -ErrorAction SilentlyContinue + } + } + + It 'builds a telemetry record with raw usage and no hardcoded cost estimate' { + $usage = [pscustomobject]@{ + prompt_tokens = 25 + completion_tokens = 15 + total_tokens = 45 + totalApiDurationMs = 2000 + } + + $record = New-CopilotTokenUsageRecord ` + -PRNumber 35677 ` + -Platform 'android' ` + -Phase 'CopilotReview' ` + -StepName 'STEP 5a: TRY-FIX' ` + -ModelName 'gpt-5.5' ` + -StartedAtUtc ([DateTimeOffset]::Parse('2026-06-05T10:00:00Z')) ` + -EndedAtUtc ([DateTimeOffset]::Parse('2026-06-05T10:00:05Z')) ` + -DurationMs 5000 ` + -TurnCount 2 ` + -ToolCount 3 ` + -FailedToolCount 1 ` + -Usage $usage ` + -OtelMetrics $null ` + -AicUsed 1030 ` + -ContextWindow 1100000 ` + -ContextWindowRaw '1.1M' ` + -ResultEventSeen $true ` + -ExitCode 0 + + $record.prNumber | Should -Be 35677 + $record.scriptPhase | Should -Be 'CopilotReview' + $record.copilotStep | Should -Be 'STEP 5a: TRY-FIX' + $record.apiDurationMs | Should -Be 2000 + $record.normalizedTokens.inputTokens | Should -Be 25 + $record.normalizedTokens.outputTokens | Should -Be 15 + $record.normalizedTokens.totalTokens | Should -Be 45 + $record.cliUsage.aicUsed | Should -Be 1030 + $record.cliUsage.contextWindow | Should -Be 1100000 + $record.cliUsage.contextWindowRaw | Should -Be '1.1M' + $record.usage.total_tokens | Should -Be 45 + $record.costEstimateAvailable | Should -Be $false + } + + It 'uses OTel token metrics when result usage has no token fields' { + $otelMetrics = [ordered]@{ + inputTokens = 500 + outputTokens = 75 + cachedInputTokens = 400 + reasoningOutputTokens = 25 + totalTokens = 575 + copilotCost = 7.5 + file = '/tmp/copilot-otel.jsonl' + } + + $record = New-CopilotTokenUsageRecord ` + -PRNumber 35677 ` + -Platform 'android' ` + -Phase 'CopilotReview' ` + -StepName 'STEP 5a: TRY-FIX' ` + -ModelName 'gpt-5.5' ` + -StartedAtUtc ([DateTimeOffset]::Parse('2026-06-05T10:00:00Z')) ` + -EndedAtUtc ([DateTimeOffset]::Parse('2026-06-05T10:00:05Z')) ` + -DurationMs 5000 ` + -TurnCount 2 ` + -ToolCount 3 ` + -FailedToolCount 0 ` + -Usage ([pscustomobject]@{ totalApiDurationMs = 1000 }) ` + -OtelMetrics $otelMetrics ` + -AicUsed $null ` + -ContextWindow $null ` + -ContextWindowRaw $null ` + -ResultEventSeen $true ` + -ExitCode 0 + + $record.normalizedTokens.inputTokens | Should -Be 500 + $record.normalizedTokens.outputTokens | Should -Be 75 + $record.normalizedTokens.cachedInputTokens | Should -Be 400 + $record.normalizedTokens.reasoningOutputTokens | Should -Be 25 + $record.normalizedTokens.totalTokens | Should -Be 575 + $record.normalizedTokens.otelFile | Should -Be '/tmp/copilot-otel.jsonl' + # aicUsed stays AIC-only (null here); the dollar cost is reported in its own field, + # never conflated into aicUsed. + $record.cliUsage.aicUsed | Should -BeNullOrEmpty + $record.cliUsage.copilotCost | Should -Be 7.5 + } } Describe 'Get-TrxResults' { @@ -235,3 +436,19 @@ Describe 'Get-DotNetTestResults (console-scrape fallback)' { (Get-DotNetTestResults -Lines @()).Count | Should -Be 0 } } + +Describe 'ConvertTo-AzdoSafeConsole' { + It 'defangs ##vso[ and ##[ logging-command prefixes' { + ConvertTo-AzdoSafeConsole '##vso[task.setvariable variable=x]y' | Should -Be '## vso[task.setvariable variable=x]y' + ConvertTo-AzdoSafeConsole '##[command]z' | Should -Be '## [command]z' + } + + It 'collapses CR/LF that could fabricate a fresh column-0 log line' { + ConvertTo-AzdoSafeConsole "safe`r##vso[task.complete]" | Should -Be 'safe ## vso[task.complete]' + ConvertTo-AzdoSafeConsole "Reviewing`n##vso[task.complete result=Succeeded;]done" | Should -Be 'Reviewing ## vso[task.complete result=Succeeded;]done' + } + + It 'leaves ordinary text untouched' { + ConvertTo-AzdoSafeConsole 'Reading file src/Foo.cs (## of total)' | Should -Be 'Reading file src/Foo.cs (## of total)' + } +} diff --git a/.github/scripts/Review-PR.ps1 b/.github/scripts/Review-PR.ps1 index 581dc40422a0..22ca57170c32 100644 --- a/.github/scripts/Review-PR.ps1 +++ b/.github/scripts/Review-PR.ps1 @@ -37,6 +37,9 @@ .PARAMETER LogFile Capture all output via Start-Transcript +.PARAMETER TokenUsageOutputDir + Directory where Copilot CLI token-usage telemetry records should be written. + .EXAMPLE .\Review-PR.ps1 -PRNumber 33687 .\Review-PR.ps1 -PRNumber 33687 -Platform ios @@ -66,7 +69,10 @@ param( [switch]$DryRun, [Parameter(Mandatory = $false)] - [string]$LogFile + [string]$LogFile, + + [Parameter(Mandatory = $false)] + [string]$TokenUsageOutputDir ) $ErrorActionPreference = 'Stop' @@ -174,6 +180,10 @@ $autonomousRules = @" $reviewBranch = "pr-review-$PRNumber" +if ([string]::IsNullOrWhiteSpace($TokenUsageOutputDir)) { + $TokenUsageOutputDir = Join-Path $RepoRoot "CustomAgentLogsTmp/PRState/$PRNumber/PRAgent/token-usage/raw" +} + # ─── Prerequisites ──────────────────────────────────────────────────────────── if ($runSetup) { Write-Host "📋 Checking prerequisites..." -ForegroundColor Yellow @@ -302,6 +312,16 @@ if ($DryRun) { Write-Host " 🔀 Merging PR commits (squashed)..." -ForegroundColor Cyan git merge --squash $tempBranch 2>&1 | Out-Null if ($LASTEXITCODE -eq 0) { + # Ensure both staged and unstaged merge output is committed. Some + # squash merges can leave tracked files modified in the worktree rather + # than only staged; Gate later requires fix files to be committed so it + # can restore them with `git checkout HEAD`. + git add -A 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + git branch -D $tempBranch 2>$null + Write-Error "Failed to stage squashed PR changes"; exit 1 + } + # Check if there's anything to commit (PR might already be merged) $staged = git diff --cached --quiet 2>$null; $hasStagedChanges = $LASTEXITCODE -ne 0 if ($hasStagedChanges) { @@ -315,6 +335,14 @@ if ($DryRun) { Write-Host " ⚠️ No changes to merge (PR may already be up to date)" -ForegroundColor Yellow } + git diff --quiet 2>$null; $hasWorktreeChanges = $LASTEXITCODE -ne 0 + git diff --cached --quiet 2>$null; $hasIndexChanges = $LASTEXITCODE -ne 0 + if ($hasWorktreeChanges -or $hasIndexChanges) { + Write-Error "Review branch has uncommitted tracked changes after setup. Gate cannot proceed safely." + git status --short + exit 1 + } + if (Get-Command Remove-StaleMauiBotIssueComments -ErrorAction SilentlyContinue) { Remove-StaleMauiBotIssueComments ` -PRNumber $PRNumber ` @@ -393,6 +421,19 @@ if ($Phase -and $Phase -ne 'Setup') { Write-Error "Setup phase did not complete (sentinel not found at '$sentinelFile'). Cannot proceed with -Phase $Phase." exit 1 } + + if (-not $DryRun) { + git checkout $reviewBranch 2>$null | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to checkout review branch '$reviewBranch' before -Phase $Phase." + exit 1 + } + git reset --hard HEAD 2>$null | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to reset review branch '$reviewBranch' before -Phase $Phase." + exit 1 + } + } } # ─── Helper: Parse `dotnet test --logger "console;verbosity=detailed"` ────── @@ -553,6 +594,468 @@ function Get-TrxResults { } } +# ─── Helper: Copilot token usage telemetry ──────────────────────────────────── +function ConvertTo-AzdoSafeConsole { + param([string]$Text) + # Collapse ALL line-break/control chars (CR/LF/FF/VT) to a space so PR-influenceable streamed + # agent output can't fabricate a fresh column-0 line, then defang AzDO logging-command prefixes + # (##vso[ / ##[). Applied to every Write-Host of streamed content (messages, intents, tool args). + return ($Text -replace '[\r\n\f\v]+', ' ') -replace '##(?=\[|vso\[)', '## ' +} + +function Test-IsNumericValue { + param([object]$Value) + + return ( + $Value -is [byte] -or + $Value -is [sbyte] -or + $Value -is [int16] -or + $Value -is [uint16] -or + $Value -is [int] -or + $Value -is [uint32] -or + $Value -is [long] -or + $Value -is [uint64] -or + $Value -is [float] -or + $Value -is [double] -or + $Value -is [decimal] + ) +} + +function Get-ObjectMemberValue { + param( + [object]$InputObject, + [string[]]$Names + ) + + if ($null -eq $InputObject) { return $null } + + foreach ($name in $Names) { + if ($InputObject -is [System.Collections.IDictionary] -and $InputObject.Contains($name)) { + return $InputObject[$name] + } + + $property = $InputObject.PSObject.Properties[$name] + if ($property) { + return $property.Value + } + } + + return $null +} + +function Get-CopilotUsageTokenFields { + param( + [object]$Value, + [string]$Path = '' + ) + + $fields = New-Object System.Collections.ArrayList + if ($null -eq $Value) { return @() } + + if (Test-IsNumericValue $Value) { + if ($Path -match '(?i)token') { + [void]$fields.Add([ordered]@{ + Path = $Path + Value = [double]$Value + }) + } + return @($fields.ToArray()) + } + + if ($Value -is [string]) { return @() } + + if ($Value -is [System.Collections.IDictionary]) { + foreach ($key in $Value.Keys) { + $childPath = if ($Path) { "$Path.$key" } else { [string]$key } + foreach ($field in Get-CopilotUsageTokenFields -Value $Value[$key] -Path $childPath) { + [void]$fields.Add($field) + } + } + return @($fields.ToArray()) + } + + if ($Value -is [System.Collections.IEnumerable]) { + $index = 0 + foreach ($item in $Value) { + $childPath = if ($Path) { "$Path[$index]" } else { "[$index]" } + foreach ($field in Get-CopilotUsageTokenFields -Value $item -Path $childPath) { + [void]$fields.Add($field) + } + $index++ + } + return @($fields.ToArray()) + } + + foreach ($property in $Value.PSObject.Properties) { + if ($property.MemberType -notin @('NoteProperty', 'Property', 'AliasProperty')) { + continue + } + + $childPath = if ($Path) { "$Path.$($property.Name)" } else { $property.Name } + foreach ($field in Get-CopilotUsageTokenFields -Value $property.Value -Path $childPath) { + [void]$fields.Add($field) + } + } + + return @($fields.ToArray()) +} + +function Get-TokenFieldSum { + param([object[]]$Fields) + + $items = @($Fields | Where-Object { $null -ne $_ }) + if ($items.Count -eq 0) { return $null } + + $sum = 0.0 + foreach ($item in $items) { + $sum += [double]$item.Value + } + + return [long][Math]::Round($sum) +} + +function Get-TokenFieldPathDepth { + param([string]$Path) + # Nesting depth = number of '.'/'[' segment separators in the dotted/indexed path. + return ([regex]::Matches([string]$Path, '[.\[]')).Count +} + +function Select-CanonicalTokenFields { + param([object[]]$Fields) + + # Prevent double-counting when a payload carries BOTH a root aggregate and a nested + # per-model breakdown for the same unit (e.g. inputTokens=1000 plus perModel[*].inputTokens + # = 600+400). Prefer the shallowest matches; only fall through to the deeper breakdown when + # no shallower aggregate exists. Flat payloads (a single depth) are unaffected. + $items = @($Fields) + if ($items.Count -le 1) { return $items } + $minDepth = ($items | ForEach-Object { Get-TokenFieldPathDepth $_.Path } | Measure-Object -Minimum).Minimum + return @($items | Where-Object { (Get-TokenFieldPathDepth $_.Path) -eq $minDepth }) +} + +function Get-CopilotTokenMetrics { + param([object]$Usage) + + $tokenFields = @(Get-CopilotUsageTokenFields -Value $Usage) + $inputFields = @($tokenFields | Where-Object { + $_.Path -match '(?i)(input|prompt)' -and + $_.Path -notmatch '(?i)(cache|cached)' -and + $_.Path -notmatch '(?i)total' + }) + $outputFields = @($tokenFields | Where-Object { + $_.Path -match '(?i)(output|completion)' -and + $_.Path -notmatch '(?i)(cache|cached)' -and + $_.Path -notmatch '(?i)total' + }) + $cachedInputFields = @($tokenFields | Where-Object { + $_.Path -match '(?i)(cache|cached)' -and + $_.Path -match '(?i)(input|prompt|read)' + }) + $explicitTotalFields = @($tokenFields | Where-Object { + $_.Path -match '(?i)total' -and + $_.Path -match '(?i)token' + }) + + $inputTokens = Get-TokenFieldSum -Fields (Select-CanonicalTokenFields $inputFields) + $outputTokens = Get-TokenFieldSum -Fields (Select-CanonicalTokenFields $outputFields) + $cachedInputTokens = Get-TokenFieldSum -Fields (Select-CanonicalTokenFields $cachedInputFields) + $totalTokens = Get-TokenFieldSum -Fields (Select-CanonicalTokenFields $explicitTotalFields) + if ($null -eq $totalTokens -and ($null -ne $inputTokens -or $null -ne $outputTokens)) { + $totalTokens = [long](($inputTokens ?? 0) + ($outputTokens ?? 0)) + } + + return [ordered]@{ + inputTokens = $inputTokens + outputTokens = $outputTokens + cachedInputTokens = $cachedInputTokens + totalTokens = $totalTokens + rawTokenFields = @($tokenFields) + } +} + +function Convert-CopilotCompactNumber { + param([string]$Value) + + if ([string]::IsNullOrWhiteSpace($Value)) { return $null } + + $normalized = ($Value -replace ',', '').Trim() + if ($normalized -notmatch '^(?[0-9]+(?:\.[0-9]+)?)\s*(?[KMGkmg])?$') { + return $null + } + + $number = [double]$Matches['number'] + $multiplier = switch ($Matches['suffix'].ToUpperInvariant()) { + 'K' { 1000 } + 'M' { 1000000 } + 'G' { 1000000000 } + default { 1 } + } + + return [long][Math]::Round($number * $multiplier) +} + +function Get-CopilotCliUsageLineData { + param([string]$Line) + + $data = [ordered]@{} + if ([string]::IsNullOrWhiteSpace($Line)) { + return $data + } + + if ($Line -match 'Session:\s*(?[0-9]+(?:\.[0-9]+)?)\s*AIC\s+used') { + $data.aicUsed = [double]$Matches['aic'] + } + + if ($Line -match '^\s*(?.+?)\s*[\u2022\u00b7]\s*(?[0-9][0-9,]*(?:\.[0-9]+)?\s*[KMGkmg]?)\s+context\s*$') { + $contextRaw = $Matches['context'].Trim() + $data.model = $Matches['model'].Trim() + $data.contextWindowRaw = $contextRaw + $data.contextWindow = Convert-CopilotCompactNumber -Value $contextRaw + } + + return $data +} + +function Get-CopilotOtelTokenMetrics { + param([string]$Path) + + $metrics = [ordered]@{ + inputTokens = $null + outputTokens = $null + cachedInputTokens = $null + reasoningOutputTokens = $null + totalTokens = $null + copilotCost = $null + available = $false + file = $Path + } + + if ([string]::IsNullOrWhiteSpace($Path) -or -not (Test-Path $Path)) { + return $metrics + } + + $spanSums = @{ + input = 0.0 + output = 0.0 + cached = 0.0 + reasoning = 0.0 + cost = 0.0 + } + $spanSeen = @{ + input = $false + output = $false + cached = $false + reasoning = $false + cost = $false + } + + $metricSums = @{ + input = 0.0 + output = 0.0 + cached = 0.0 + } + $metricSeen = @{ + input = $false + output = $false + cached = $false + } + + foreach ($line in Get-Content -Path $Path -Encoding UTF8) { + if ([string]::IsNullOrWhiteSpace($line)) { continue } + + try { + $entry = $line | ConvertFrom-Json -ErrorAction Stop + } catch { + continue + } + + if ($entry.type -eq 'span' -and $entry.attributes) { + $attributes = $entry.attributes + $inputValue = Get-ObjectMemberValue -InputObject $attributes -Names @('gen_ai.usage.input_tokens') + $outputValue = Get-ObjectMemberValue -InputObject $attributes -Names @('gen_ai.usage.output_tokens') + $cachedValue = Get-ObjectMemberValue -InputObject $attributes -Names @('gen_ai.usage.cache_read.input_tokens', 'gen_ai.usage.cache_read_input_tokens') + $reasoningValue = Get-ObjectMemberValue -InputObject $attributes -Names @('gen_ai.usage.reasoning.output_tokens', 'gen_ai.usage.reasoning_output_tokens') + $costValue = Get-ObjectMemberValue -InputObject $attributes -Names @('github.copilot.cost') + + if (Test-IsNumericValue $inputValue) { $spanSums.input += [double]$inputValue; $spanSeen.input = $true } + if (Test-IsNumericValue $outputValue) { $spanSums.output += [double]$outputValue; $spanSeen.output = $true } + if (Test-IsNumericValue $cachedValue) { $spanSums.cached += [double]$cachedValue; $spanSeen.cached = $true } + if (Test-IsNumericValue $reasoningValue) { $spanSums.reasoning += [double]$reasoningValue; $spanSeen.reasoning = $true } + if (Test-IsNumericValue $costValue) { $spanSums.cost += [double]$costValue; $spanSeen.cost = $true } + } elseif ($entry.type -eq 'metric' -and $entry.name -eq 'gen_ai.client.token.usage') { + foreach ($point in @($entry.dataPoints)) { + $tokenType = [string](Get-ObjectMemberValue -InputObject $point.attributes -Names @('gen_ai.token.type')) + $sumValue = Get-ObjectMemberValue -InputObject $point.value -Names @('sum') + if (-not (Test-IsNumericValue $sumValue)) { continue } + + if ($tokenType -eq 'input') { + $metricSums.input += [double]$sumValue + $metricSeen.input = $true + } elseif ($tokenType -eq 'output') { + $metricSums.output += [double]$sumValue + $metricSeen.output = $true + } elseif ($tokenType -match '(?i)cache') { + $metricSums.cached += [double]$sumValue + $metricSeen.cached = $true + } + } + } + } + + $inputTokens = if ($spanSeen.input) { [long][Math]::Round($spanSums.input) } elseif ($metricSeen.input) { [long][Math]::Round($metricSums.input) } else { $null } + $outputTokens = if ($spanSeen.output) { [long][Math]::Round($spanSums.output) } elseif ($metricSeen.output) { [long][Math]::Round($metricSums.output) } else { $null } + $cachedInputTokens = if ($spanSeen.cached) { [long][Math]::Round($spanSums.cached) } elseif ($metricSeen.cached) { [long][Math]::Round($metricSums.cached) } else { $null } + $reasoningOutputTokens = if ($spanSeen.reasoning) { [long][Math]::Round($spanSums.reasoning) } else { $null } + $copilotCost = if ($spanSeen.cost) { [Math]::Round($spanSums.cost, 3) } else { $null } + + $totalTokens = if ($null -ne $inputTokens -or $null -ne $outputTokens) { + [long](($inputTokens ?? 0) + ($outputTokens ?? 0)) + } else { + $null + } + + $metrics.inputTokens = $inputTokens + $metrics.outputTokens = $outputTokens + $metrics.cachedInputTokens = $cachedInputTokens + $metrics.reasoningOutputTokens = $reasoningOutputTokens + $metrics.totalTokens = $totalTokens + $metrics.copilotCost = $copilotCost + $metrics.available = ($null -ne $inputTokens -or $null -ne $outputTokens -or $null -ne $cachedInputTokens -or $null -ne $copilotCost) + + return $metrics +} + +function New-CopilotTokenUsageRecord { + param( + [int]$PRNumber, + [string]$Platform, + [string]$Phase, + [string]$StepName, + [string]$ModelName, + [datetimeoffset]$StartedAtUtc, + [datetimeoffset]$EndedAtUtc, + [long]$DurationMs, + [int]$TurnCount, + [int]$ToolCount, + [int]$FailedToolCount, + [object]$Usage, + [object]$OtelMetrics, + [object]$AicUsed, + [object]$ContextWindow, + [string]$ContextWindowRaw, + [bool]$ResultEventSeen, + [int]$ExitCode + ) + + $apiDurationValue = Get-ObjectMemberValue -InputObject $Usage -Names @('totalApiDurationMs', 'total_api_duration_ms') + $apiDurationMs = if (Test-IsNumericValue $apiDurationValue) { [long]$apiDurationValue } else { $null } + $usageTokenMetrics = Get-CopilotTokenMetrics -Usage $Usage + + $inputTokens = $usageTokenMetrics.inputTokens + $outputTokens = $usageTokenMetrics.outputTokens + $cachedInputTokens = $usageTokenMetrics.cachedInputTokens + $totalTokens = $usageTokenMetrics.totalTokens + $reasoningOutputTokens = $null + $copilotCost = $null + $otelFile = $null + + if ($OtelMetrics) { + if ($null -eq $inputTokens -and $null -ne $OtelMetrics.inputTokens) { $inputTokens = $OtelMetrics.inputTokens } + if ($null -eq $outputTokens -and $null -ne $OtelMetrics.outputTokens) { $outputTokens = $OtelMetrics.outputTokens } + if ($null -eq $cachedInputTokens -and $null -ne $OtelMetrics.cachedInputTokens) { $cachedInputTokens = $OtelMetrics.cachedInputTokens } + if ($null -eq $totalTokens -and $null -ne $OtelMetrics.totalTokens) { $totalTokens = $OtelMetrics.totalTokens } + $reasoningOutputTokens = $OtelMetrics.reasoningOutputTokens + $copilotCost = $OtelMetrics.copilotCost + $otelFile = $OtelMetrics.file + } + + # Keep billing units separate — never fall back across unit types (AIC credits vs dollar + # cost vs request count). Collapsing them into one field produces meaningless aggregate + # sums (credits + dollars + counts) for the downstream consumer. + $aicUsed = $AicUsed + $premiumRequests = Get-ObjectMemberValue -InputObject $Usage -Names @('premiumRequests') + if (Test-IsNumericValue $premiumRequests) { + $premiumRequests = [double]$premiumRequests + } else { + $premiumRequests = $null + } + + return [ordered]@{ + schemaVersion = 1 + generatedAtUtc = ([DateTimeOffset]::UtcNow).ToString('o') + prNumber = $PRNumber + platform = $Platform + pipeline = [ordered]@{ + buildId = $env:BUILD_BUILDID + buildNumber = $env:BUILD_BUILDNUMBER + definitionName = $env:BUILD_DEFINITIONNAME + stageName = $env:SYSTEM_STAGENAME + jobName = $env:SYSTEM_JOBNAME + jobDisplayName = $env:SYSTEM_JOBDISPLAYNAME + taskInstanceId = $env:SYSTEM_TASKINSTANCEID + } + scriptPhase = if ($Phase) { $Phase } else { 'All' } + copilotStep = $StepName + model = $ModelName + startedAtUtc = $StartedAtUtc.ToString('o') + endedAtUtc = $EndedAtUtc.ToString('o') + durationMs = $DurationMs + apiDurationMs = $apiDurationMs + resultEventSeen = $ResultEventSeen + exitCode = $ExitCode + turnCount = $TurnCount + toolCount = $ToolCount + failedToolCount = $FailedToolCount + cliUsage = [ordered]@{ + aicUsed = $aicUsed + copilotCost = $copilotCost + premiumRequests = $premiumRequests + contextWindow = $ContextWindow + contextWindowRaw = $ContextWindowRaw + } + normalizedTokens = [ordered]@{ + inputTokens = $inputTokens + outputTokens = $outputTokens + cachedInputTokens = $cachedInputTokens + reasoningOutputTokens = $reasoningOutputTokens + totalTokens = $totalTokens + rawTokenFields = @($usageTokenMetrics.rawTokenFields) + otelFile = $otelFile + } + usage = $Usage + costEstimateAvailable = $false + costEstimateNote = 'Dollar cost not calculated; no trusted rate table configured.' + } +} + +function Write-CopilotTokenUsageRecord { + param( + [string]$OutputDir, + [object]$Record + ) + + if ([string]::IsNullOrWhiteSpace($OutputDir) -or $null -eq $Record) { + return + } + + try { + New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null + $stepName = [string]$Record.copilotStep + $safeStepName = ($stepName -replace '[^A-Za-z0-9._-]+', '-').Trim('-') + if ([string]::IsNullOrWhiteSpace($safeStepName)) { + $safeStepName = 'copilot-step' + } + + $timestamp = [DateTimeOffset]::UtcNow.ToString('yyyyMMddTHHmmssfffZ') + $fileName = "copilot-token-usage-$timestamp-$safeStepName-$([guid]::NewGuid().ToString('N')).json" + $path = Join-Path $OutputDir $fileName + $Record | ConvertTo-Json -Depth 50 | Set-Content -Path $path -Encoding UTF8 + Write-Host " Token usage record: $path" -ForegroundColor DarkGray + } catch { + Write-Host " WARNING: Failed to write Copilot token usage record: $_" -ForegroundColor Yellow + } +} + # ─── Helper: Invoke Copilot ────────────────────────────────────────────────── function Invoke-CopilotStep { param([string]$StepName, [string]$Prompt) @@ -568,12 +1071,18 @@ function Invoke-CopilotStep { return 0 } + $startedAtUtc = [DateTimeOffset]::UtcNow $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() $toolCount = 0 $turnCount = 0 $currentIntent = "" $modelName = "" $failedTools = @() + $resultEventSeen = $false + $resultUsage = $null + $cliAicUsed = $null + $cliContextWindow = $null + $cliContextWindowRaw = $null # Tool icon mapping for common tools $toolIcons = @{ @@ -592,133 +1101,212 @@ function Invoke-CopilotStep { # Model is overridable via $env:COPILOT_REVIEW_MODEL so contributors without internal-model access # can run this script (e.g., with 'claude-opus-4.6' or 'claude-sonnet-4.6'). $copilotModel = if ($env:COPILOT_REVIEW_MODEL) { $env:COPILOT_REVIEW_MODEL } else { 'gpt-5.5' } - & copilot -p $Prompt --allow-all --output-format json --model $copilotModel --secret-env-vars=GH_TOKEN,COPILOT_GITHUB_TOKEN,GITHUB_TOKEN 2>&1 | ForEach-Object { - $line = $_.ToString() - try { - $event = $line | ConvertFrom-Json -ErrorAction Stop - switch ($event.type) { - 'session.tools_updated' { - if ($event.data.model) { - $modelName = $event.data.model - Write-Host " ⚙️ Model: " -ForegroundColor DarkGray -NoNewline - Write-Host $modelName -ForegroundColor DarkCyan + if ([string]::IsNullOrWhiteSpace($modelName)) { + $modelName = $copilotModel + } + $safeOtelStepName = ($StepName -replace '[^A-Za-z0-9._-]+', '-').Trim('-') + if ([string]::IsNullOrWhiteSpace($safeOtelStepName)) { + $safeOtelStepName = 'copilot-step' + } + $otelPath = $null + if (-not [string]::IsNullOrWhiteSpace($TokenUsageOutputDir)) { + New-Item -ItemType Directory -Path $TokenUsageOutputDir -Force | Out-Null + $otelPath = Join-Path $TokenUsageOutputDir "copilot-otel-$([DateTimeOffset]::UtcNow.ToString('yyyyMMddTHHmmssfffZ'))-$safeOtelStepName-$([guid]::NewGuid().ToString('N')).jsonl" + } + + $savedOtel = @{ + COPILOT_OTEL_FILE_EXPORTER_PATH = $env:COPILOT_OTEL_FILE_EXPORTER_PATH + COPILOT_OTEL_EXPORTER_TYPE = $env:COPILOT_OTEL_EXPORTER_TYPE + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = $env:OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT + } + try { + if ($otelPath) { + $env:COPILOT_OTEL_FILE_EXPORTER_PATH = $otelPath + $env:COPILOT_OTEL_EXPORTER_TYPE = 'file' + $env:OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = 'false' + } + + & copilot -p $Prompt --allow-all --output-format json --model $copilotModel --secret-env-vars=GH_TOKEN,COPILOT_GITHUB_TOKEN,GITHUB_TOKEN 2>&1 | ForEach-Object { + $line = $_.ToString() + try { + $event = $line | ConvertFrom-Json -ErrorAction Stop + switch ($event.type) { + 'session.tools_updated' { + if ($event.data.model) { + $modelName = $event.data.model + Write-Host " ⚙️ Model: " -ForegroundColor DarkGray -NoNewline + Write-Host $modelName -ForegroundColor DarkCyan + } } - } - 'assistant.turn_start' { - $turnCount++ - $elapsed = $stopwatch.Elapsed.ToString("mm\:ss") - Write-Host "" - Write-Host " ┌─ Turn $turnCount " -ForegroundColor DarkGray -NoNewline - Write-Host "[$elapsed]" -ForegroundColor DarkYellow -NoNewline - if ($currentIntent) { - Write-Host " · $currentIntent" -ForegroundColor DarkCyan - } else { + 'assistant.turn_start' { + $turnCount++ + $elapsed = $stopwatch.Elapsed.ToString("mm\:ss") Write-Host "" + Write-Host " ┌─ Turn $turnCount " -ForegroundColor DarkGray -NoNewline + Write-Host "[$elapsed]" -ForegroundColor DarkYellow -NoNewline + if ($currentIntent) { + Write-Host " · $currentIntent" -ForegroundColor DarkCyan + } else { + Write-Host "" + } } - } - 'assistant.turn_end' { - Write-Host " └─" -ForegroundColor DarkGray - } - 'tool.execution_start' { - $toolName = $event.data.toolName - $args_ = $event.data.arguments - - # Capture intent changes silently - if ($toolName -eq 'report_intent') { - $currentIntent = $args_.intent ?? $currentIntent - Write-Host " │ 🎯 " -ForegroundColor DarkGray -NoNewline - Write-Host $currentIntent -ForegroundColor Yellow - break + 'assistant.turn_end' { + Write-Host " └─" -ForegroundColor DarkGray } + 'tool.execution_start' { + $toolName = $event.data.toolName + $args_ = $event.data.arguments + + # Capture intent changes silently + if ($toolName -eq 'report_intent') { + # Sanitize once at the store so every later echo (incl. the + # assistant.turn_start " · $currentIntent" line) inherits the safe value. + $currentIntent = ConvertTo-AzdoSafeConsole ($args_.intent ?? $currentIntent) + Write-Host " │ 🎯 " -ForegroundColor DarkGray -NoNewline + Write-Host $currentIntent -ForegroundColor Yellow + break + } - $toolCount++ - $icon = $toolIcons[$toolName] - if (-not $icon) { - # Prefix match for github-mcp-server-* and other compound names - $icon = if ($toolName -like 'github-*') { '🔀' } else { '🔧' } - } + $toolCount++ + $icon = $toolIcons[$toolName] + if (-not $icon) { + # Prefix match for github-mcp-server-* and other compound names + $icon = if ($toolName -like 'github-*') { '🔀' } else { '🔧' } + } - # Build a short display name for long tool names - $displayName = $toolName -replace '^github-mcp-server-', 'gh/' + # Build a short display name for long tool names + $displayName = ConvertTo-AzdoSafeConsole ($toolName -replace '^github-mcp-server-', 'gh/') - # Pick the most useful detail from arguments - $detail = $args_.description ?? $args_.intent ?? '' - if (-not $detail) { - # Fallback: pick first informative arg - $detail = $args_.command ?? $args_.pattern ?? $args_.query ?? $args_.path ?? $args_.prompt ?? '' - } - if ($detail) { - $detail = $detail.Substring(0, [Math]::Min($detail.Length, 90)) - # Truncate at last word boundary if we cut mid-word - if ($detail.Length -eq 90) { - $lastSpace = $detail.LastIndexOf(' ') - if ($lastSpace -gt 60) { $detail = $detail.Substring(0, $lastSpace) + "…" } - else { $detail += "…" } + # Pick the most useful detail from arguments + $detail = $args_.description ?? $args_.intent ?? '' + if (-not $detail) { + # Fallback: pick first informative arg + $detail = $args_.command ?? $args_.pattern ?? $args_.query ?? $args_.path ?? $args_.prompt ?? '' + } + if ($detail) { + $detail = $detail.Substring(0, [Math]::Min($detail.Length, 90)) + # Truncate at last word boundary if we cut mid-word + if ($detail.Length -eq 90) { + $lastSpace = $detail.LastIndexOf(' ') + if ($lastSpace -gt 60) { $detail = $detail.Substring(0, $lastSpace) + "…" } + else { $detail += "…" } + } } - } - Write-Host " │ $icon " -ForegroundColor DarkGray -NoNewline - Write-Host $displayName -ForegroundColor Cyan -NoNewline - if ($detail) { - Write-Host " $detail" -ForegroundColor DarkGray - } else { - Write-Host "" + Write-Host " │ $icon " -ForegroundColor DarkGray -NoNewline + Write-Host $displayName -ForegroundColor Cyan -NoNewline + if ($detail) { + Write-Host " $(ConvertTo-AzdoSafeConsole $detail)" -ForegroundColor DarkGray + } else { + Write-Host "" + } } - } - 'tool.execution_complete' { - if (-not $event.data.success) { - $failedTool = $event.data.toolCallId - $failedTools += $failedTool - Write-Host " │ ❌ Tool failed" -ForegroundColor Red + 'tool.execution_complete' { + if (-not $event.data.success) { + $failedTool = $event.data.toolCallId + $failedTools += $failedTool + Write-Host " │ ❌ Tool failed" -ForegroundColor Red + } } - } - 'assistant.message' { - $content = $event.data.content - # Show agent text responses (skip empty tool-request-only messages) - if ($content -and $content.Trim()) { - $preview = $content.Trim() - if ($preview.Length -gt 400) { - $preview = $preview.Substring(0, 400) + "…" + 'assistant.message' { + $content = $event.data.content + # Show agent text responses (skip empty tool-request-only messages) + if ($content -and $content.Trim()) { + $preview = $content.Trim() + if ($preview.Length -gt 400) { + $preview = $preview.Substring(0, 400) + "…" + } + # Agent message content is PR-influenceable; defang AzDO logging-command + # prefixes + strip CR before echoing so it can't inject a pipeline command. + $preview = ConvertTo-AzdoSafeConsole $preview + Write-Host " │ 💬 " -ForegroundColor DarkGray -NoNewline + Write-Host $preview -ForegroundColor White } - Write-Host " │ 💬 " -ForegroundColor DarkGray -NoNewline - Write-Host $preview -ForegroundColor White } - } - 'result' { - # Final stats — note: 'result' is a top-level event with no 'data' wrapper. - $usage = $event.usage - if ($usage) { - $elapsed = $stopwatch.Elapsed.ToString("mm\:ss") - $apiMs = if ($usage.totalApiDurationMs) { [math]::Round($usage.totalApiDurationMs / 1000, 1) } else { "?" } - $changes = $usage.codeChanges - $filesChanged = if ($changes -and $changes.filesModified) { @($changes.filesModified).Count } else { 0 } - $linesAdded = if ($changes) { $changes.linesAdded } else { 0 } - $linesRemoved = if ($changes) { $changes.linesRemoved } else { 0 } - - Write-Host "" - Write-Host " ╭──────────────────────────────────────────╮" -ForegroundColor DarkGray - Write-Host " │ ⏱ $elapsed elapsed ($($apiMs)s API)" -ForegroundColor DarkGray -NoNewline - Write-Host " │ 🔧 $toolCount tools" -ForegroundColor DarkGray -NoNewline - Write-Host " │ 🔄 $turnCount turns" -ForegroundColor DarkGray - if ($filesChanged -gt 0 -or $linesAdded -gt 0 -or $linesRemoved -gt 0) { - Write-Host " │ 📝 $filesChanged files " -ForegroundColor DarkGray -NoNewline - Write-Host "+$linesAdded" -ForegroundColor Green -NoNewline - Write-Host "/" -ForegroundColor DarkGray -NoNewline - Write-Host "-$linesRemoved" -ForegroundColor Red + 'result' { + # Final stats — note: 'result' is a top-level event with no 'data' wrapper. + $resultEventSeen = $true + $usage = $event.usage + $resultUsage = $usage + if ($usage) { + $elapsed = $stopwatch.Elapsed.ToString("mm\:ss") + $apiMs = if ($usage.totalApiDurationMs) { [math]::Round($usage.totalApiDurationMs / 1000, 1) } else { "?" } + $changes = $usage.codeChanges + $filesChanged = if ($changes -and $changes.filesModified) { @($changes.filesModified).Count } else { 0 } + $linesAdded = if ($changes) { $changes.linesAdded } else { 0 } + $linesRemoved = if ($changes) { $changes.linesRemoved } else { 0 } + + Write-Host "" + Write-Host " ╭──────────────────────────────────────────╮" -ForegroundColor DarkGray + Write-Host " │ ⏱ $elapsed elapsed ($($apiMs)s API)" -ForegroundColor DarkGray -NoNewline + Write-Host " │ 🔧 $toolCount tools" -ForegroundColor DarkGray -NoNewline + Write-Host " │ 🔄 $turnCount turns" -ForegroundColor DarkGray + if ($filesChanged -gt 0 -or $linesAdded -gt 0 -or $linesRemoved -gt 0) { + Write-Host " │ 📝 $filesChanged files " -ForegroundColor DarkGray -NoNewline + Write-Host "+$linesAdded" -ForegroundColor Green -NoNewline + Write-Host "/" -ForegroundColor DarkGray -NoNewline + Write-Host "-$linesRemoved" -ForegroundColor Red + } + Write-Host " ╰──────────────────────────────────────────╯" -ForegroundColor DarkGray } - Write-Host " ╰──────────────────────────────────────────╯" -ForegroundColor DarkGray } } + } catch { + $cliLineData = Get-CopilotCliUsageLineData -Line $line + if ($cliLineData.Contains('aicUsed')) { + $cliAicUsed = $cliLineData.aicUsed + } + if ($cliLineData.Contains('contextWindow')) { + $cliContextWindow = $cliLineData.contextWindow + $cliContextWindowRaw = $cliLineData.contextWindowRaw + } + if ($cliLineData.Contains('model') -and -not [string]::IsNullOrWhiteSpace([string]$cliLineData.model)) { + $modelName = [string]$cliLineData.model + } + + # Non-JSON line (e.g. stats) — strip CR and defang any AzDO logging-command + # prefix (##vso[ / ##[) so PR-influenced Copilot output can't inject a + # pipeline command (e.g. "\r##vso[task.setvariable...]"), then echo as-is. + if ($line.Trim()) { + $safeLine = ($line -replace "`r", '') -replace '##(?=\[|vso\[)', '## ' + Write-Host " $safeLine" -ForegroundColor DarkGray + } } - } catch { - # Non-JSON line (e.g. stats) — pass through as-is - if ($line.Trim()) { - Write-Host " $line" -ForegroundColor DarkGray + } + } finally { + foreach ($key in $savedOtel.Keys) { + if ($null -eq $savedOtel[$key]) { + Remove-Item -Path ("env:" + $key) -ErrorAction SilentlyContinue + } else { + Set-Item -Path ("env:" + $key) -Value $savedOtel[$key] } } } $exitCode = $LASTEXITCODE $stopwatch.Stop() + $endedAtUtc = [DateTimeOffset]::UtcNow + $otelMetrics = Get-CopilotOtelTokenMetrics -Path $otelPath + + $usageRecord = New-CopilotTokenUsageRecord ` + -PRNumber $PRNumber ` + -Platform $Platform ` + -Phase $Phase ` + -StepName $StepName ` + -ModelName $modelName ` + -StartedAtUtc $startedAtUtc ` + -EndedAtUtc $endedAtUtc ` + -DurationMs $stopwatch.ElapsedMilliseconds ` + -TurnCount $turnCount ` + -ToolCount $toolCount ` + -FailedToolCount (@($failedTools).Count) ` + -Usage $resultUsage ` + -OtelMetrics $otelMetrics ` + -AicUsed $cliAicUsed ` + -ContextWindow $cliContextWindow ` + -ContextWindowRaw $cliContextWindowRaw ` + -ResultEventSeen $resultEventSeen ` + -ExitCode $exitCode + Write-CopilotTokenUsageRecord -OutputDir $TokenUsageOutputDir -Record $usageRecord if ($exitCode -eq 0) { Write-Host " ✅ $StepName completed" -ForegroundColor Green @@ -1058,6 +1646,22 @@ for ($gateAttempt = 1; $gateAttempt -le $maxGateAttempts; $gateAttempt++) { if ($gateAttempt -gt 1) { Write-Host " 🔄 Retry $gateAttempt/$maxGateAttempts — previous attempt hit environment error" -ForegroundColor Yellow } + if (-not $DryRun) { + # Each verification attempt mutates fix files while testing the without-fix + # state. If an attempt aborts before restoring those files, retries must + # start from the committed review branch or they fail immediately with + # "uncommitted changes detected in fix files". + git checkout $reviewBranch 2>$null | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to checkout review branch '$reviewBranch' before gate attempt $gateAttempt." + exit 1 + } + git reset --hard HEAD 2>$null | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to reset review branch '$reviewBranch' before gate attempt $gateAttempt." + exit 1 + } + } # Clear previous attempt's report so a crash mid-run doesn't leak its classification into this one. Remove-Item $gateContentFile -Force -ErrorAction SilentlyContinue # Note: -RequireFullVerification is intentionally OMITTED. The verify script @@ -1263,6 +1867,10 @@ $gateVerdictDir = if ($TrustedScriptsDir) { $d } $gateResult | Set-Content (Join-Path $gateVerdictDir "gate-result.txt") -Encoding UTF8 +# Also persist into PRAgent/gate (always overwritten = trusted), which ships in the CopilotLogs +# artifact — the UpdateAISummaryComment APPROVE-veto reads the result from there in CI (the +# $gateVerdictDir copy above can land at the staging root, which the artifact does not include). +$gateResult | Set-Content (Join-Path $gateOutputDir "gate-result.txt") -Encoding UTF8 Write-Host " 📄 Gate result persisted: $gateResult" -ForegroundColor Gray # Persist regression data for CopilotReview phase (try-fix instructions) @@ -1355,35 +1963,6 @@ $gateStatusForPrompt = switch ($gateResult) { default { "Gate ❌ FAILED — tests did NOT behave as expected." } } -$rerunContextInstruction = "" -$rerunContextPath = Join-Path $RepoRoot "CustomAgentLogsTmp/PRState/$PRNumber/PRAgent/rerun/context.md" -$rerunContextScript = Join-Path $ScriptsDir "Resolve-RerunEligibility.ps1" -if (Test-Path $rerunContextScript) { - try { - Write-Host "Generating deterministic rerun context..." -ForegroundColor Cyan - & pwsh -NoProfile -File $rerunContextScript ` - -PRNumber $PRNumber ` - -Owner 'dotnet' ` - -Repo 'maui' ` - -ContextOutputPath $rerunContextPath - if ($LASTEXITCODE -eq 0 -and (Test-Path $rerunContextPath)) { - Write-Host " ✅ rerun context: $rerunContextPath" -ForegroundColor Green - $rerunContextInstruction = @" - -## Deterministic rerun context - -Before pre-flight, read ``CustomAgentLogsTmp/PRState/$PRNumber/PRAgent/rerun/context.md`` if it exists. This file is generated without AI and lists new comments/commits since the latest AI Summary or previous ``/review rerun`` checkpoint. - -When the file has new activity, explicitly include a "New activity since previous AI Summary" subsection in ``pre-flight/content.md`` and prioritize that delta when deciding what changed since the previous review. -"@ - } else { - Write-Host " ⚠️ rerun context generation exited with code $LASTEXITCODE" -ForegroundColor Yellow - } - } catch { - Write-Host " ⚠️ rerun context generation failed: $_" -ForegroundColor Yellow - } -} - # Build regression test instruction for try-fix candidates $regressionTestInstruction = "" if ($risksData -and $regressionTests -and $regressionTests.Count -gt 0) { @@ -1417,7 +1996,6 @@ Generate alternative fix candidates for PR #$PRNumber using an iterative expert- ## Phase 1 — Pre-Flight (context only) Use the pr-review skill's pre-flight phase to gather context about the issue and PR. Do NOT modify code. Write summary to ``CustomAgentLogsTmp/PRState/$PRNumber/PRAgent/pre-flight/content.md``. -$rerunContextInstruction ## Phase 2 — Iterative Try-Fix loop For each candidate, follow this cycle: diff --git a/.github/scripts/Review-Tests.ps1 b/.github/scripts/Review-Tests.ps1 index 419687889709..bb35dbc79384 100644 --- a/.github/scripts/Review-Tests.ps1 +++ b/.github/scripts/Review-Tests.ps1 @@ -255,6 +255,7 @@ function New-TestFailureReviewBody { else { "> Test-failure review results are available based on commit [``$commitSha7``]($commitUrl)." } + $authorPing += ' To request a fresh review after new comments, commits, or CI runs, comment `/review tests`.' $badges = $badgeLines -join "`n" @@ -264,7 +265,6 @@ $marker ## Tests Failure Analysis $authorPing -> To request a fresh review after new comments, commits, or CI runs, comment ``/review tests``.

$badges diff --git a/.github/scripts/post-ai-summary-comment.ps1 b/.github/scripts/post-ai-summary-comment.ps1 index cb22f4de9481..7048123cb469 100644 --- a/.github/scripts/post-ai-summary-comment.ps1 +++ b/.github/scripts/post-ai-summary-comment.ps1 @@ -390,6 +390,34 @@ function Test-HasNonPRWinner { } } +function Test-RunValidationFailed { + param([Parameter(Mandatory = $true)][string]$PRAgentDir) + + # Gate: key off the trusted gate-result.txt, which Review-PR.ps1 always overwrites with the + # real $gateResult. Do NOT parse gate/content.md — it is not always cleaned before the gate + # runs, so a PR could commit a forged "Gate Result: PASSED" content.md to bypass the veto. + $gateResultFile = Join-Path $PRAgentDir 'gate/gate-result.txt' + if (Test-Path -LiteralPath $gateResultFile) { + $gateResult = (Get-Content -Raw -LiteralPath $gateResultFile -Encoding UTF8).Trim() + if ($gateResult -match '(?im)^FAILED$') { return $true } + } + + # UI tests: the pipeline render writes "❌ **Deep UI tests** — N passed, M failed …" with no + # "Result:" line, so detect the failure icon on a bold test header or a non-zero "N failed" + # count ("marked failed by TRX" on the passing branch has no digit immediately before + # "failed", so it does not match). Skip the "no UI tests needed" no-op placeholder. + $uiFile = Join-Path $PRAgentDir 'uitests/content.md' + if (Test-Path -LiteralPath $uiFile) { + $uiContent = Get-Content -Raw -LiteralPath $uiFile -Encoding UTF8 + if (-not (Test-PhaseContentIsNoOp -PhaseKey 'uitests' -Content $uiContent) -and + ($uiContent -match '(?im)❌\s*\*\*[^*\n]*tests\*\*' -or $uiContent -match '(?im)\b[1-9]\d*\s+failed\b')) { + return $true + } + } + + return $false +} + function Get-AIReviewEventForRun { param( [string]$ReportContent, @@ -399,6 +427,13 @@ function Get-AIReviewEventForRun { ) $reviewEvent = Get-AIReviewEvent -ReportContent $ReportContent + + # Validation veto: never post an APPROVE review over a failed gate / device-test validation, + # even when the report body recommends APPROVE (the report can be stale vs. current-run results). + if ($reviewEvent -eq 'APPROVE' -and (Test-RunValidationFailed -PRAgentDir $PRAgentDir)) { + return 'REQUEST_CHANGES' + } + if ((Test-HasNonPRWinner -PRAgentDir $PRAgentDir) -and $reviewEvent -eq 'COMMENT') { return 'REQUEST_CHANGES' } @@ -519,8 +554,6 @@ try { Write-Host "⚠️ Failed to fetch commit info: $_" -ForegroundColor Yellow $commitJson = $null } -$commitTitle = if ($commitJson) { ($commitJson.message -split "`n")[0] } else { "Unknown" } -$commitTitle = $commitTitle -replace '&','&' -replace '<','<' -replace '>','>' $commitSha7 = if ($commitJson) { $commitJson.sha.Substring(0, 7) } else { "unknown" } $commitFull = if ($commitJson) { $commitJson.sha } else { "" } $commitUrl = if ($commitJson) { "https://github.com/dotnet/maui/commit/$commitFull" } else { "#" } @@ -592,7 +625,7 @@ if ($existingRaw) { $authorPing = "" if ($prAuthor) { - $authorPing = "> @$prAuthor — new AI review results are available based on this last commit: $commitSha7.`n> **$commitTitle**" + $authorPing = "> @$prAuthor — new AI review results are available based on this last commit: $commitSha7." $authorPing += ' To request a fresh review after new comments or commits, comment `/review rerun`.' } diff --git a/.github/scripts/shared/Aggregate-CopilotTokenUsage.ps1 b/.github/scripts/shared/Aggregate-CopilotTokenUsage.ps1 new file mode 100644 index 000000000000..2dbcbcad07a8 --- /dev/null +++ b/.github/scripts/shared/Aggregate-CopilotTokenUsage.ps1 @@ -0,0 +1,420 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Aggregates Copilot CLI usage telemetry into publishable artifacts. + +.DESCRIPTION + Reads raw telemetry records emitted by Review-PR.ps1 and writes JSON, + Markdown, CSV, and JSONL summaries. Missing input is treated as a valid + no-usage report so the publishing stage can still produce artifacts after + partial pipeline failures. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$InputRoot, + + [Parameter(Mandatory = $true)] + [string]$OutputDir, + + [Parameter(Mandatory = $false)] + [string]$PRNumber, + + [Parameter(Mandatory = $false)] + [string[]]$ExpectedStages = @( + 'ReviewPR', + 'RunDeepUITests', + 'UpdateAISummaryComment', + 'AnalyzeCopilotTokenUsage' + ) +) + +$ErrorActionPreference = 'Stop' + +function Get-ObjectMemberValue { + param( + [object]$InputObject, + [string[]]$Names + ) + + if ($null -eq $InputObject) { return $null } + + foreach ($name in $Names) { + if ($InputObject -is [System.Collections.IDictionary] -and $InputObject.Contains($name)) { + return $InputObject[$name] + } + + $property = $InputObject.PSObject.Properties[$name] + if ($property) { + return $property.Value + } + } + + return $null +} + +function Get-NestedValue { + param( + [object]$InputObject, + [string[]]$Path + ) + + $current = $InputObject + foreach ($segment in $Path) { + $current = Get-ObjectMemberValue -InputObject $current -Names @($segment) + if ($null -eq $current) { return $null } + } + + return $current +} + +function Get-NumericOrNull { + param([object]$Value) + + if ($null -eq $Value) { return $null } + if ($Value -is [byte] -or + $Value -is [sbyte] -or + $Value -is [int16] -or + $Value -is [uint16] -or + $Value -is [int] -or + $Value -is [uint32] -or + $Value -is [long] -or + $Value -is [uint64] -or + $Value -is [float] -or + $Value -is [double] -or + $Value -is [decimal]) { + return [double]$Value + } + + $parsed = 0.0 + if ([double]::TryParse([string]$Value, + [System.Globalization.NumberStyles]::Float -bor [System.Globalization.NumberStyles]::AllowThousands, + [System.Globalization.CultureInfo]::InvariantCulture, + [ref]$parsed)) { + return $parsed + } + + return $null +} + +function Get-NullableSum { + param([object[]]$Values) + + $hasValue = $false + $sum = 0.0 + foreach ($value in @($Values)) { + $numeric = Get-NumericOrNull -Value $value + if ($null -ne $numeric) { + $hasValue = $true + $sum += $numeric + } + } + + if (-not $hasValue) { return $null } + return [long][Math]::Round($sum) +} + +function Get-NullableDecimalSum { + param([object[]]$Values) + + $hasValue = $false + $sum = 0.0 + foreach ($value in @($Values)) { + $numeric = Get-NumericOrNull -Value $value + if ($null -ne $numeric) { + $hasValue = $true + $sum += $numeric + } + } + + if (-not $hasValue) { return $null } + return [Math]::Round($sum, 3) +} + +function Get-RecordStageName { + param([object]$Record) + + $stageName = [string](Get-NestedValue -InputObject $Record -Path @('pipeline', 'stageName')) + if ([string]::IsNullOrWhiteSpace($stageName)) { + return 'ReviewPR' + } + + return $stageName +} + +function Get-RecordTokenValue { + param( + [object]$Record, + [string]$Name + ) + + return Get-NestedValue -InputObject $Record -Path @('normalizedTokens', $Name) +} + +function Get-RecordAicUsed { + param([object]$Record) + + return Get-NestedValue -InputObject $Record -Path @('cliUsage', 'aicUsed') +} + +function Get-RecordCopilotCost { + param([object]$Record) + + return Get-NestedValue -InputObject $Record -Path @('cliUsage', 'copilotCost') +} + +function Get-RecordPremiumRequests { + param([object]$Record) + + return Get-NestedValue -InputObject $Record -Path @('cliUsage', 'premiumRequests') +} + +function Read-CopilotTokenUsageRecords { + param([string]$Root) + + $records = New-Object System.Collections.ArrayList + if ([string]::IsNullOrWhiteSpace($Root) -or -not (Test-Path $Root)) { + return @() + } + + # Security: the CopilotLogs artifact also bundles PR-worktree content (CustomAgentLogsTmp) + # where PR-controlled steps can drop forged copilot-token-usage-*.json. Only trust records + # under the pipeline-written 'copilot-token-usage/raw' subtree, and never those under + # CustomAgentLogsTmp, so a forged file can't be aggregated and dispatched as official usage. + $files = Get-ChildItem -Path $Root -Recurse -File -Filter 'copilot-token-usage-*.json' -ErrorAction SilentlyContinue | + Where-Object { + $normalized = $_.FullName -replace '\\', '/' + $normalized -match '/copilot-token-usage/raw/' -and + $normalized -notmatch '/CustomAgentLogsTmp/' -and + $normalized -notmatch '/agent-pr-session/' + } | + Sort-Object FullName + + foreach ($file in @($files)) { + try { + $record = Get-Content -Path $file.FullName -Raw -Encoding UTF8 | ConvertFrom-Json -ErrorAction Stop + $record | Add-Member -NotePropertyName sourceFile -NotePropertyValue $file.FullName -Force + [void]$records.Add($record) + } catch { + Write-Warning "Skipping malformed token usage record '$($file.FullName)': $_" + } + } + + return @($records.ToArray()) +} + +function New-StageSummaryRows { + param( + [object[]]$Records, + [string[]]$ExpectedStages + ) + + $stageNames = New-Object System.Collections.ArrayList + foreach ($stage in @($ExpectedStages)) { + if (-not [string]::IsNullOrWhiteSpace($stage) -and -not $stageNames.Contains($stage)) { + [void]$stageNames.Add($stage) + } + } + + foreach ($record in @($Records)) { + $stage = Get-RecordStageName -Record $record + if (-not $stageNames.Contains($stage)) { + [void]$stageNames.Add($stage) + } + } + + $rows = New-Object System.Collections.ArrayList + foreach ($stage in @($stageNames.ToArray())) { + $stageRecords = @($Records | Where-Object { (Get-RecordStageName -Record $_) -eq $stage }) + $hasRecords = $stageRecords.Count -gt 0 + [void]$rows.Add([pscustomobject][ordered]@{ + stageName = $stage + invocationCount = $stageRecords.Count + inputTokens = if ($hasRecords) { Get-NullableSum -Values @($stageRecords | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'inputTokens' }) } else { 0 } + outputTokens = if ($hasRecords) { Get-NullableSum -Values @($stageRecords | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'outputTokens' }) } else { 0 } + cachedInputTokens = if ($hasRecords) { Get-NullableSum -Values @($stageRecords | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'cachedInputTokens' }) } else { 0 } + reasoningOutputTokens = if ($hasRecords) { Get-NullableSum -Values @($stageRecords | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'reasoningOutputTokens' }) } else { 0 } + totalTokens = if ($hasRecords) { Get-NullableSum -Values @($stageRecords | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'totalTokens' }) } else { 0 } + aicUsed = if ($hasRecords) { Get-NullableDecimalSum -Values @($stageRecords | ForEach-Object { Get-RecordAicUsed -Record $_ }) } else { 0 } + copilotCost = if ($hasRecords) { Get-NullableDecimalSum -Values @($stageRecords | ForEach-Object { Get-RecordCopilotCost -Record $_ }) } else { 0 } + premiumRequests = if ($hasRecords) { Get-NullableDecimalSum -Values @($stageRecords | ForEach-Object { Get-RecordPremiumRequests -Record $_ }) } else { 0 } + durationMs = if ($hasRecords) { Get-NullableSum -Values @($stageRecords | ForEach-Object { $_.durationMs }) } else { 0 } + apiDurationMs = if ($hasRecords) { Get-NullableSum -Values @($stageRecords | ForEach-Object { $_.apiDurationMs }) } else { 0 } + turnCount = if ($hasRecords) { Get-NullableSum -Values @($stageRecords | ForEach-Object { $_.turnCount }) } else { 0 } + toolCount = if ($hasRecords) { Get-NullableSum -Values @($stageRecords | ForEach-Object { $_.toolCount }) } else { 0 } + note = if ($hasRecords) { '' } else { 'No Copilot invocation observed in this stage.' } + }) + } + + return @($rows.ToArray()) +} + +function New-StepSummaryRows { + param([object[]]$Records) + + $groups = @{} + foreach ($record in @($Records)) { + $stage = Get-RecordStageName -Record $record + $step = [string]$record.copilotStep + $model = [string]$record.model + $key = "$stage|$step|$model" + if (-not $groups.ContainsKey($key)) { + $groups[$key] = New-Object System.Collections.ArrayList + } + [void]$groups[$key].Add($record) + } + + $rows = New-Object System.Collections.ArrayList + foreach ($key in ($groups.Keys | Sort-Object)) { + $items = @($groups[$key].ToArray()) + $first = $items[0] + [void]$rows.Add([pscustomobject][ordered]@{ + stageName = Get-RecordStageName -Record $first + scriptPhase = [string]$first.scriptPhase + copilotStep = [string]$first.copilotStep + model = [string]$first.model + invocationCount = $items.Count + inputTokens = Get-NullableSum -Values @($items | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'inputTokens' }) + outputTokens = Get-NullableSum -Values @($items | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'outputTokens' }) + cachedInputTokens = Get-NullableSum -Values @($items | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'cachedInputTokens' }) + reasoningOutputTokens = Get-NullableSum -Values @($items | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'reasoningOutputTokens' }) + totalTokens = Get-NullableSum -Values @($items | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'totalTokens' }) + aicUsed = Get-NullableDecimalSum -Values @($items | ForEach-Object { Get-RecordAicUsed -Record $_ }) + copilotCost = Get-NullableDecimalSum -Values @($items | ForEach-Object { Get-RecordCopilotCost -Record $_ }) + premiumRequests = Get-NullableDecimalSum -Values @($items | ForEach-Object { Get-RecordPremiumRequests -Record $_ }) + durationMs = Get-NullableSum -Values @($items | ForEach-Object { $_.durationMs }) + apiDurationMs = Get-NullableSum -Values @($items | ForEach-Object { $_.apiDurationMs }) + turnCount = Get-NullableSum -Values @($items | ForEach-Object { $_.turnCount }) + toolCount = Get-NullableSum -Values @($items | ForEach-Object { $_.toolCount }) + }) + } + + return @($rows.ToArray()) +} + +function New-CopilotTokenUsageSummary { + param( + [object[]]$Records, + [string[]]$ExpectedStages, + [string]$PRNumber + ) + + $stageRows = @(New-StageSummaryRows -Records $Records -ExpectedStages $ExpectedStages) + $stepRows = @(New-StepSummaryRows -Records $Records) + + return [ordered]@{ + schemaVersion = 1 + generatedAtUtc = ([DateTimeOffset]::UtcNow).ToString('o') + prNumber = $PRNumber + costEstimateAvailable = $false + costEstimateNote = 'Dollar cost not calculated; no trusted rate table configured.' + recordCount = @($Records).Count + expectedStages = @($ExpectedStages) + totals = [ordered]@{ + invocationCount = @($Records).Count + inputTokens = Get-NullableSum -Values @($Records | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'inputTokens' }) + outputTokens = Get-NullableSum -Values @($Records | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'outputTokens' }) + cachedInputTokens = Get-NullableSum -Values @($Records | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'cachedInputTokens' }) + reasoningOutputTokens = Get-NullableSum -Values @($Records | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'reasoningOutputTokens' }) + totalTokens = Get-NullableSum -Values @($Records | ForEach-Object { Get-RecordTokenValue -Record $_ -Name 'totalTokens' }) + aicUsed = Get-NullableDecimalSum -Values @($Records | ForEach-Object { Get-RecordAicUsed -Record $_ }) + copilotCost = Get-NullableDecimalSum -Values @($Records | ForEach-Object { Get-RecordCopilotCost -Record $_ }) + premiumRequests = Get-NullableDecimalSum -Values @($Records | ForEach-Object { Get-RecordPremiumRequests -Record $_ }) + durationMs = Get-NullableSum -Values @($Records | ForEach-Object { $_.durationMs }) + apiDurationMs = Get-NullableSum -Values @($Records | ForEach-Object { $_.apiDurationMs }) + turnCount = Get-NullableSum -Values @($Records | ForEach-Object { $_.turnCount }) + toolCount = Get-NullableSum -Values @($Records | ForEach-Object { $_.toolCount }) + } + stages = @($stageRows) + steps = @($stepRows) + } +} + +function Format-UsageValue { + param([object]$Value) + + if ($null -eq $Value -or [string]::IsNullOrWhiteSpace([string]$Value)) { + return 'n/a' + } + + return [string]$Value +} + +function New-CopilotTokenUsageMarkdown { + param([object]$Summary) + + $lines = New-Object System.Collections.ArrayList + [void]$lines.Add('# Copilot token usage') + [void]$lines.Add('') + [void]$lines.Add("- PR: $(if ($Summary.prNumber) { $Summary.prNumber } else { 'n/a' })") + [void]$lines.Add("- Records: $($Summary.recordCount)") + [void]$lines.Add("- Cost estimate: not calculated (no trusted rate table configured)") + [void]$lines.Add('') + [void]$lines.Add('## Totals') + [void]$lines.Add('') + [void]$lines.Add('| Metric | Value |') + [void]$lines.Add('|---|---:|') + [void]$lines.Add("| Invocations | $($Summary.totals.invocationCount) |") + [void]$lines.Add("| Input tokens | $(Format-UsageValue $Summary.totals.inputTokens) |") + [void]$lines.Add("| Output tokens | $(Format-UsageValue $Summary.totals.outputTokens) |") + [void]$lines.Add("| Cached input tokens | $(Format-UsageValue $Summary.totals.cachedInputTokens) |") + [void]$lines.Add("| Reasoning output tokens | $(Format-UsageValue $Summary.totals.reasoningOutputTokens) |") + [void]$lines.Add("| Total tokens | $(Format-UsageValue $Summary.totals.totalTokens) |") + [void]$lines.Add("| AIC used | $(Format-UsageValue $Summary.totals.aicUsed) |") + [void]$lines.Add("| Copilot cost (USD) | $(Format-UsageValue $Summary.totals.copilotCost) |") + [void]$lines.Add("| Premium requests | $(Format-UsageValue $Summary.totals.premiumRequests) |") + [void]$lines.Add("| Elapsed ms | $(Format-UsageValue $Summary.totals.durationMs) |") + [void]$lines.Add("| API duration ms | $(Format-UsageValue $Summary.totals.apiDurationMs) |") + [void]$lines.Add("| Turns | $(Format-UsageValue $Summary.totals.turnCount) |") + [void]$lines.Add("| Tools | $(Format-UsageValue $Summary.totals.toolCount) |") + [void]$lines.Add('') + [void]$lines.Add('## By stage') + [void]$lines.Add('') + [void]$lines.Add('| Stage | Invocations | Input | Output | Cached input | Reasoning | Total | AIC used | Elapsed ms | API ms | Note |') + [void]$lines.Add('|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|') + foreach ($stage in @($Summary.stages)) { + [void]$lines.Add("| $($stage.stageName) | $($stage.invocationCount) | $(Format-UsageValue $stage.inputTokens) | $(Format-UsageValue $stage.outputTokens) | $(Format-UsageValue $stage.cachedInputTokens) | $(Format-UsageValue $stage.reasoningOutputTokens) | $(Format-UsageValue $stage.totalTokens) | $(Format-UsageValue $stage.aicUsed) | $(Format-UsageValue $stage.durationMs) | $(Format-UsageValue $stage.apiDurationMs) | $($stage.note) |") + } + [void]$lines.Add('') + [void]$lines.Add('## By Copilot step') + [void]$lines.Add('') + if (@($Summary.steps).Count -eq 0) { + [void]$lines.Add('No Copilot invocations were recorded.') + } else { + [void]$lines.Add('| Stage | Phase | Step | Model | Invocations | Input | Output | Cached input | Reasoning | Total | AIC used | Elapsed ms | API ms |') + [void]$lines.Add('|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|') + foreach ($step in @($Summary.steps)) { + [void]$lines.Add("| $($step.stageName) | $($step.scriptPhase) | $($step.copilotStep) | $($step.model) | $($step.invocationCount) | $(Format-UsageValue $step.inputTokens) | $(Format-UsageValue $step.outputTokens) | $(Format-UsageValue $step.cachedInputTokens) | $(Format-UsageValue $step.reasoningOutputTokens) | $(Format-UsageValue $step.totalTokens) | $(Format-UsageValue $step.aicUsed) | $(Format-UsageValue $step.durationMs) | $(Format-UsageValue $step.apiDurationMs) |") + } + } + + return ($lines -join [Environment]::NewLine) + [Environment]::NewLine +} + +New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null + +$records = @(Read-CopilotTokenUsageRecords -Root $InputRoot) +$summary = New-CopilotTokenUsageSummary -Records $records -ExpectedStages $ExpectedStages -PRNumber $PRNumber + +$rawJsonlPath = Join-Path $OutputDir 'token-usage-raw.jsonl' +if ($records.Count -gt 0) { + $records | ForEach-Object { $_ | ConvertTo-Json -Depth 50 -Compress } | + Set-Content -Path $rawJsonlPath -Encoding UTF8 +} else { + '' | Set-Content -Path $rawJsonlPath -Encoding UTF8 +} + +$summary | ConvertTo-Json -Depth 50 | Set-Content -Path (Join-Path $OutputDir 'token-usage-summary.json') -Encoding UTF8 +New-CopilotTokenUsageMarkdown -Summary $summary | Set-Content -Path (Join-Path $OutputDir 'token-usage-summary.md') -Encoding UTF8 + +$csvPath = Join-Path $OutputDir 'token-usage-by-step.csv' +if (@($summary.steps).Count -gt 0) { + @($summary.steps) | Export-Csv -Path $csvPath -NoTypeInformation -Encoding UTF8 +} else { + 'stageName,scriptPhase,copilotStep,model,invocationCount,inputTokens,outputTokens,cachedInputTokens,reasoningOutputTokens,totalTokens,aicUsed,copilotCost,premiumRequests,durationMs,apiDurationMs,turnCount,toolCount' | + Set-Content -Path $csvPath -Encoding UTF8 +} + +Write-Host "Copilot token usage records: $($summary.recordCount)" +Write-Host "Copilot token usage artifact directory: $OutputDir" diff --git a/.github/skills/review-test-failures/SKILL.md b/.github/skills/review-test-failures/SKILL.md index bceff0448ef0..9a1a6a1b0287 100644 --- a/.github/skills/review-test-failures/SKILL.md +++ b/.github/skills/review-test-failures/SKILL.md @@ -104,8 +104,7 @@ Use a compact PR conversation comment body. Start with a stable marker, put the ## Tests Failure Analysis -> @[PR author] — test-failure review results are available based on commit [`[sha7]`]([commit URL]). -> To request a fresh review after new comments, commits, or CI runs, comment `/review tests`. +> @[PR author] — test-failure review results are available based on commit [`[sha7]`]([commit URL]). To request a fresh review after new comments, commits, or CI runs, comment `/review tests`.

Overall [verdict] diff --git a/.github/skills/verify-tests-fail-without-fix/scripts/verify-tests-fail.ps1 b/.github/skills/verify-tests-fail-without-fix/scripts/verify-tests-fail.ps1 index e54d900a1cf1..42238872a9d7 100644 --- a/.github/skills/verify-tests-fail-without-fix/scripts/verify-tests-fail.ps1 +++ b/.github/skills/verify-tests-fail-without-fix/scripts/verify-tests-fail.ps1 @@ -533,15 +533,29 @@ function Invoke-TestRunWithRetry { if ($attempt -lt $MaxRetries) { Write-Host " ⚠️ Environment error (attempt $attempt/$MaxRetries): $($result.Error) — retrying in 30s..." -ForegroundColor Yellow - # On app launch failures, reboot the simulator/emulator to recover - if ($result.Error -match "APP_LAUNCH_FAILURE|exit code.*83|app.*crash" -and $script:BootedDeviceUdid -and $script:BootedDeviceUdid -ne "host") { - Write-Host " 🔄 Rebooting device ($($script:BootedDeviceUdid)) to recover from app launch failure..." -ForegroundColor Yellow + # Device test environment failures can leave the emulator/simulator in + # a bad package-manager state for the next without/with-fix attempt. + if ($result.Error -match "APP_LAUNCH_FAILURE|exit code.*83|app.*crash|package.*install|package.*operation|command timed out|XHarness exit 78" -and $script:BootedDeviceUdid -and $script:BootedDeviceUdid -ne "host") { + Write-Host " 🔄 Rebooting device ($($script:BootedDeviceUdid)) to recover from environment error: $($result.Error)" -ForegroundColor Yellow if ($Platform -in @("ios", "catalyst", "maccatalyst")) { xcrun simctl shutdown $script:BootedDeviceUdid 2>$null - Start-Sleep -Seconds 5 - xcrun simctl boot $script:BootedDeviceUdid 2>$null + # Boot and block until the simulator has finished booting (services ready), + # not just powered on, before the next attempt. + xcrun simctl bootstatus $script:BootedDeviceUdid -b 2>$null } elseif ($Platform -eq "android") { adb -s $script:BootedDeviceUdid reboot 2>$null + adb -s $script:BootedDeviceUdid wait-for-device 2>$null + # wait-for-device only waits for adbd to respond; the package manager, + # installer and launcher aren't ready until boot actually completes, so + # poll sys.boot_completed + bootanim (up to 180s) before retrying — + # otherwise the next attempt hits the same install/launch failure. + $bootDeadline = (Get-Date).AddSeconds(180) + while ((Get-Date) -lt $bootDeadline) { + $bootCompleted = (adb -s $script:BootedDeviceUdid shell getprop sys.boot_completed 2>$null | Out-String).Trim() + $bootAnim = (adb -s $script:BootedDeviceUdid shell getprop init.svc.bootanim 2>$null | Out-String).Trim() + if ($bootCompleted -eq '1' -and $bootAnim -eq 'stopped') { break } + Start-Sleep -Seconds 3 + } } } @@ -632,6 +646,9 @@ function Get-TestResultFromOutput { $envErrorPatterns = @( @{ Pattern = "error ADB0010.*InstallFailedException"; Message = "App install failed (ADB broken pipe)" } @{ Pattern = "XHarness exit code:\s*83"; Message = "App failed to launch (XHarness exit 83)" } + @{ Pattern = "XHarness exit code:\s*78"; Message = "Package installation failed (XHarness exit 78)" } + @{ Pattern = "PACKAGE_INSTALLATION_FAILURE"; Message = "Package installation failed (XHarness package installation failure)" } + @{ Pattern = "Waiting for command timed out: execution may be compromised"; Message = "Device package operation timed out" } @{ Pattern = "Application test run crashed"; Message = "App crashed during test run" } @{ Pattern = "SIGABRT.*load_aot_module"; Message = "App crashed during AOT loading" } @{ Pattern = "AppiumServerHasNotBeenStartedLocally"; Message = "Appium server failed to start" } diff --git a/.github/workflows/copilot-review-tests.lock.yml b/.github/workflows/copilot-review-tests.lock.yml index 5fd05e31d8e0..a9b37cded086 100644 --- a/.github/workflows/copilot-review-tests.lock.yml +++ b/.github/workflows/copilot-review-tests.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"53de81b2269a74e2ed986e4b5dde3d9548eb6501470a563a38241db242b0fe66","body_hash":"44becb5921a41041d2bfe4f01ab40a77a42cabe36e4b2f68d213eba783e7bde3","compiler_version":"v0.77.5","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.6"} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"53de81b2269a74e2ed986e4b5dde3d9548eb6501470a563a38241db242b0fe66","body_hash":"720072ecd8974d07a6087a6ac32a5916f8c8ab281a84fdfb13145c96dd66c63c","compiler_version":"v0.77.5","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.6"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"3ea13c02d765410340d533515cb31a7eef2baaf0","version":"v0.77.5"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.58"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.22"},{"image":"ghcr.io/github/github-mcp-server:v1.1.0"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} # ___ _ _ # / _ \ | | (_) diff --git a/.github/workflows/copilot-review-tests.md b/.github/workflows/copilot-review-tests.md index 49d1d7c68701..4887b799fe4d 100644 --- a/.github/workflows/copilot-review-tests.md +++ b/.github/workflows/copilot-review-tests.md @@ -215,8 +215,7 @@ If dry-run mode is not active, call `add_comment` exactly once with `item_number ## Tests Failure Analysis -> @[PR author] — test-failure review results are available based on commit [`[sha7]`]([commit URL]). -> To request a fresh review after new comments, commits, or CI runs, comment `/review tests`. +> @[PR author] — test-failure review results are available based on commit [`[sha7]`]([commit URL]). To request a fresh review after new comments, commits, or CI runs, comment `/review tests`.

Overall [verdict] diff --git a/eng/pipelines/ci-copilot.yml b/eng/pipelines/ci-copilot.yml index 3a0fe03a6d8c..77de024c94eb 100644 --- a/eng/pipelines/ci-copilot.yml +++ b/eng/pipelines/ci-copilot.yml @@ -110,6 +110,72 @@ stages: env: PARAM_PR_NUMBER: ${{ parameters.PRNumber }} + # ───────────────────────────────────────────────────────── + # Resolve the PR's target (base) branch and switch the + # worktree to it BEFORE installing workloads or merging. + # + # This pipeline branch (self) may be based on net11.0 while + # a PR targets main (net10), or vice-versa. Two things derive + # from the checked-out tree and must follow the PR's REAL base: + # 1. Workloads / SDK band — build.ps1 --target=dotnet reads + # global.json + eng/Versions.props from the worktree. + # 2. Merge base — Review-PR.ps1 -Phase Setup squash-merges + # the PR onto the current HEAD in CI. + # Running on the wrong base ⇒ wrong workloads + cross-branch + # squash-merge conflicts. Switching to the PR's base fixes both, + # so ONE pipeline branch handles main- and net11-targeting PRs. + # + # Trusted scripts are captured HERE (from the pipeline ref, + # before the base-branch checkout) so later tasks keep running + # the reviewed pipeline-branch .github/scripts even though the + # worktree is swapped to the base branch (security rule 3). + # ───────────────────────────────────────────────────────── + - bash: | + set -e + + # Capture trusted scripts from the pipeline ref (self) BEFORE + # switching branches — later tasks invoke from $TRUSTED only. + TRUSTED="$(Build.ArtifactStagingDirectory)/trusted-github" + chmod -R u+w "$TRUSTED" 2>/dev/null || true + rm -rf "$TRUSTED" 2>/dev/null || true + mkdir -p "$TRUSTED" + cp -r .github/scripts "$TRUSTED/scripts" + cp -r .github/skills "$TRUSTED/skills" + cp -r eng/scripts "$TRUSTED/eng-scripts" + chmod -R a-w "$TRUSTED" + echo "Trusted scripts copied to $TRUSTED" + + # PRNumber is validated upstream ('Validate Parameters'); re-guard + # before use in a gh call. + if ! [[ "${PARAM_PR_NUMBER}" =~ ^[1-9][0-9]*$ ]]; then + echo "##vso[task.logissue type=error]PRNumber must be a positive integer" + exit 1 + fi + + # Detect the PR's base branch (GitHub-provided metadata; repo + # inferred from the origin remote, same as Review-PR.ps1). + BASE_REF=$(gh pr view "${PARAM_PR_NUMBER}" --json baseRefName -q .baseRefName) + + # Allowlist: only ever switch to a known protected base branch. + # Validate BEFORE echoing/using to avoid any injection via the ref. + if ! [[ "${BASE_REF}" =~ ^(main|net[0-9]+\.0)$ ]]; then + echo "##vso[task.logissue type=error]Unexpected PR base branch (expected main or netN.0). Refusing to switch." + exit 1 + fi + echo "PR #${PARAM_PR_NUMBER} targets base branch: ${BASE_REF}" + + # Switch the worktree to the PR's base branch (detached, to avoid + # 'branch already checked out in another worktree' conflicts). + git fetch origin "${BASE_REF}" --no-tags + git checkout --detach "origin/${BASE_REF}" + echo "Worktree now at $(git rev-parse --short HEAD) (origin/${BASE_REF}) for workloads + merge base" + name: ResolveBaseBranch + displayName: 'Resolve PR base branch (workloads + merge base)' + retryCountOnTaskFailure: 2 + env: + GH_TOKEN: $(GH_COMMENT_TOKEN) + PARAM_PR_NUMBER: ${{ parameters.PRNumber }} + # Enable KVM for Android emulator on Linux (same as ui-tests-steps.yml / device-tests-steps.yml) - ${{ if eq(parameters.Platform, 'android') }}: - template: common/enable-kvm.yml @@ -614,25 +680,22 @@ stages: # Create artifacts directory mkdir -p $(Build.ArtifactStagingDirectory)/copilot-logs + mkdir -p $(Build.ArtifactStagingDirectory)/copilot-token-usage/raw - # Copy trusted scripts from the checked-out commit so later tasks - # (which may be on a merged/modified worktree) use the same .github/ - # files that were reviewed and approved on main. + # Trusted scripts were already captured from the pipeline ref in the + # 'Resolve PR base branch' step (before the base-branch checkout), so + # later tasks keep running the reviewed pipeline-branch scripts even + # though the worktree is now on the PR's base branch (security rule 3). TRUSTED="$(Build.ArtifactStagingDirectory)/trusted-github" - mkdir -p "$TRUSTED" - cp -r .github/scripts "$TRUSTED/scripts" - cp -r .github/skills "$TRUSTED/skills" - cp -r eng/scripts "$TRUSTED/eng-scripts" - chmod -R a-w "$TRUSTED" - echo "Trusted scripts copied to $TRUSTED" - # Run Setup phase (branch checkout + PR merge) + # Run Setup phase (branch checkout + PR merge) from the trusted copy. set +e - pwsh -NoProfile .github/scripts/Review-PR.ps1 \ + pwsh -NoProfile "$TRUSTED/scripts/Review-PR.ps1" \ -PRNumber "${PARAM_PR_NUMBER}" \ -Platform "${{ parameters.Platform }}" \ -Phase Setup \ -TrustedScriptsDir "$TRUSTED" \ + -TokenUsageOutputDir "$(Build.ArtifactStagingDirectory)/copilot-token-usage/raw" \ -LogFile "$(Build.ArtifactStagingDirectory)/copilot-logs/copilot_review_output.md" SETUP_EXIT=$? set -e @@ -663,6 +726,7 @@ stages: -Platform "${{ parameters.Platform }}" \ -Phase Gate \ -TrustedScriptsDir "$TRUSTED" \ + -TokenUsageOutputDir "$(Build.ArtifactStagingDirectory)/copilot-token-usage/raw" \ -LogFile "$(Build.ArtifactStagingDirectory)/copilot-logs/copilot_review_output.md" GATE_EXIT=$? set -e @@ -694,6 +758,7 @@ stages: -Platform "${{ parameters.Platform }}" \ -Phase CopilotReview \ -TrustedScriptsDir "$TRUSTED" \ + -TokenUsageOutputDir "$(Build.ArtifactStagingDirectory)/copilot-token-usage/raw" \ -LogFile "$(Build.ArtifactStagingDirectory)/copilot-logs/copilot_review_output.md" REVIEW_EXIT=$? set -e @@ -724,6 +789,7 @@ stages: -Platform "${{ parameters.Platform }}" \ -Phase Post \ -TrustedScriptsDir "$TRUSTED" \ + -TokenUsageOutputDir "$(Build.ArtifactStagingDirectory)/copilot-token-usage/raw" \ -LogFile "$(Build.ArtifactStagingDirectory)/copilot-logs/copilot_review_output.md" POST_EXIT=$? set -e @@ -761,6 +827,13 @@ stages: Copy-Item -Path ".github/agent-pr-session" -Destination $logsDir -Recurse -Force -ErrorAction SilentlyContinue } + # Copilot token usage raw records + $tokenUsageDir = "$(Build.ArtifactStagingDirectory)/copilot-token-usage" + if (Test-Path $tokenUsageDir) { + Write-Host "Copying copilot-token-usage..." + Copy-Item -Path $tokenUsageDir -Destination (Join-Path $logsDir "copilot-token-usage") -Recurse -Force -ErrorAction SilentlyContinue + } + # Review_Feedback files Get-ChildItem -Path . -Filter "Review_Feedback_*.md" -Recurse -ErrorAction SilentlyContinue | ForEach-Object { Copy-Item $_.FullName $logsDir -ErrorAction SilentlyContinue } @@ -862,6 +935,31 @@ stages: fetchDepth: 0 persistCredentials: false + # Switch the worktree to the PR's base branch before installing + # workloads and merging the PR for deep UI tests — same rationale as + # the CopilotReview stage's 'Resolve PR base branch' step (correct SDK + # band + conflict-free squash-merge regardless of the pipeline branch). + - bash: | + set -e + if ! [[ "${PARAM_PR_NUMBER}" =~ ^[1-9][0-9]*$ ]]; then + echo "##vso[task.logissue type=error]PRNumber must be a positive integer" + exit 1 + fi + BASE_REF=$(gh pr view "${PARAM_PR_NUMBER}" --json baseRefName -q .baseRefName) + if ! [[ "${BASE_REF}" =~ ^(main|net[0-9]+\.0)$ ]]; then + echo "##vso[task.logissue type=error]Unexpected PR base branch (expected main or netN.0). Refusing to switch." + exit 1 + fi + echo "PR #${PARAM_PR_NUMBER} targets base branch: ${BASE_REF}" + git fetch origin "${BASE_REF}" --no-tags + git checkout --detach "origin/${BASE_REF}" + echo "Worktree now at $(git rev-parse --short HEAD) (origin/${BASE_REF})" + displayName: 'Resolve PR base branch (workloads + merge base)' + retryCountOnTaskFailure: 2 + env: + GH_TOKEN: $(GH_COMMENT_TOKEN) + PARAM_PR_NUMBER: ${{ parameters.PRNumber }} + # Bring in .NET + workloads + tasks DLL — same prerequisites the # CopilotReview job used. Reusing the install-dotnet template # keeps the SDK version pinned to global.json. @@ -1699,6 +1797,7 @@ stages: env: GH_TOKEN: $(GH_COMMENT_TOKEN) + - stage: CleanupReviewLock displayName: 'Cleanup review lock' dependsOn: @@ -1731,3 +1830,144 @@ stages: env: GH_TOKEN: $(GH_COMMENT_TOKEN) PARAM_PR_NUMBER: ${{ parameters.PRNumber }} + + - stage: AnalyzeCopilotTokenUsage + displayName: 'Analyze Copilot token usage' + dependsOn: + - ReviewPR + - RunDeepUITests + - UpdateAISummaryComment + condition: always() + variables: + reviewPrResult: $[ dependencies.ReviewPR.result ] + runDeepUITestsResult: $[ dependencies.RunDeepUITests.result ] + updateAISummaryCommentResult: $[ dependencies.UpdateAISummaryComment.result ] + jobs: + - job: AnalyzeTokenUsage + displayName: 'Publish Copilot token usage artifact' + pool: + name: Azure Pipelines + vmImage: ubuntu-22.04 + timeoutInMinutes: 10 + steps: + - checkout: self + persistCredentials: false + + - task: DownloadPipelineArtifact@2 + displayName: 'Download CopilotLogs' + inputs: + buildType: 'current' + artifactName: 'CopilotLogs' + targetPath: '$(Pipeline.Workspace)/CopilotLogs' + continueOnError: true + + - pwsh: | + $ErrorActionPreference = 'Stop' + $inputRoot = "$(Pipeline.Workspace)/CopilotLogs" + $outputDir = "$(Build.ArtifactStagingDirectory)/copilot-token-usage" + $script = ".github/scripts/shared/Aggregate-CopilotTokenUsage.ps1" + if (-not (Test-Path $script)) { throw "$script missing" } + + & $script ` + -InputRoot $inputRoot ` + -OutputDir $outputDir ` + -PRNumber "${{ parameters.PRNumber }}" ` + -ExpectedStages @( + 'ReviewPR', + 'RunDeepUITests', + 'UpdateAISummaryComment', + 'AnalyzeCopilotTokenUsage' + ) + displayName: 'Aggregate Copilot token usage' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish CopilotTokenUsage' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)/copilot-token-usage' + artifact: 'CopilotTokenUsage' + publishLocation: 'pipeline' + condition: always() + + - pwsh: | + $ErrorActionPreference = 'Stop' + + if ([string]::IsNullOrWhiteSpace($env:VIGILANT_GUIDE_DISPATCH_TOKEN)) { + Write-Host "##[warning]Agent statistics dispatch token is not configured; skipping update dispatch." + exit 0 + } + + $summaryPath = Join-Path "${env:BUILD_ARTIFACTSTAGINGDIRECTORY}/copilot-token-usage" "token-usage-summary.json" + if (-not (Test-Path $summaryPath)) { + Write-Host "##[warning]Copilot token usage summary was not produced at $summaryPath; skipping agent statistics dispatch." + exit 0 + } + + $summary = Get-Content -Path $summaryPath -Raw -Encoding UTF8 | ConvertFrom-Json + $stageResults = [ordered]@{ + ReviewPR = "${env:REVIEW_PR_RESULT}" + RunDeepUITests = "${env:RUN_DEEP_UI_TESTS_RESULT}" + UpdateAISummaryComment = "${env:UPDATE_AI_SUMMARY_COMMENT_RESULT}" + AnalyzeCopilotTokenUsage = "${env:AGENT_JOB_STATUS}" + } + + $pipelineResult = 'succeeded' + if ($stageResults.Values -contains 'Failed') { + $pipelineResult = 'failed' + } elseif ($stageResults.Values -contains 'SucceededWithIssues') { + $pipelineResult = 'partiallySucceeded' + } + + $payload = [ordered]@{ + event_type = 'maui-copilot-token-usage' + client_payload = [ordered]@{ + schemaVersion = 1 + build = [ordered]@{ + id = "${env:BUILD_BUILD_ID}" + number = "${env:BUILD_BUILD_NUMBER}" + url = "${env:SYSTEM_COLLECTION_URI}${env:SYSTEM_TEAM_PROJECT}/_build/results?buildId=${env:BUILD_BUILD_ID}" + sourceBranch = "${env:BUILD_SOURCE_BRANCH}" + sourceVersion = "${env:BUILD_SOURCE_VERSION}" + requestedFor = "${env:BUILD_REQUESTED_FOR}" + prNumber = "${{ parameters.PRNumber }}" + platform = "${{ parameters.Platform }}" + result = $pipelineResult + stageResults = $stageResults + } + summary = $summary + } + } + + $headers = @{ + Authorization = "Bearer $env:VIGILANT_GUIDE_DISPATCH_TOKEN" + Accept = 'application/vnd.github+json' + 'X-GitHub-Api-Version' = '2022-11-28' + } + $body = $payload | ConvertTo-Json -Depth 100 -Compress + Invoke-RestMethod ` + -Method Post ` + -Uri 'https://api.github.com/repos/dotnet/maui-vigilant-guide/dispatches' ` + -Headers $headers ` + -ContentType 'application/json' ` + -Body $body | Out-Null + + Write-Host "Dispatched Copilot token usage summary to dotnet/maui-vigilant-guide for build ${env:BUILD_BUILD_ID}." + displayName: 'Dispatch agent statistics update' + condition: always() + continueOnError: true + env: + VIGILANT_GUIDE_DISPATCH_TOKEN: $(VIGILANT_GUIDE_DISPATCH_TOKEN) + # Pass AzDO macros via env (runtime data) rather than inlining $(...) into the + # pwsh script, where a value containing a quote, $(...) or backtick could break + # out of the string literal and execute in a step holding the dispatch token. + BUILD_ARTIFACTSTAGINGDIRECTORY: $(Build.ArtifactStagingDirectory) + REVIEW_PR_RESULT: $(reviewPrResult) + RUN_DEEP_UI_TESTS_RESULT: $(runDeepUITestsResult) + UPDATE_AI_SUMMARY_COMMENT_RESULT: $(updateAISummaryCommentResult) + AGENT_JOB_STATUS: $(Agent.JobStatus) + BUILD_BUILD_ID: $(Build.BuildId) + BUILD_BUILD_NUMBER: $(Build.BuildNumber) + SYSTEM_COLLECTION_URI: $(System.CollectionUri) + SYSTEM_TEAM_PROJECT: $(System.TeamProject) + BUILD_SOURCE_BRANCH: $(Build.SourceBranch) + BUILD_SOURCE_VERSION: $(Build.SourceVersion) + BUILD_REQUESTED_FOR: $(Build.RequestedFor)