Converge /review tests with azdo-build-investigator via shared CI facts doc

.github/scripts/Review-Tests.ps1

@@ -164,10 +164,15 @@ function Get-VerdictColor {
     param([string]$Verdict)
 
     switch -Regex ($Verdict) {
-        'Likely PR-caused' { return 'd1242f' }
-        'Likely unrelated' { return '1a7f37' }
+        # Overall merge-readiness verdicts
+        'Ready to merge' { return '1a7f37' }
         'No failures found' { return '1a7f37' }
+        'Not ready' { return 'd1242f' }
         'Insufficient data' { return '6e7781' }
+        'Needs human' { return 'bf8700' }
+        # Backward-compatible per-failure verdict words
+        'Likely PR-caused' { return 'd1242f' }
+        'Likely unrelated' { return '1a7f37' }
         default { return 'bf8700' }
     }
 }
@@ -230,11 +235,17 @@ function New-TestFailureReviewBody {
     $verdictColor = Get-VerdictColor -Verdict $verdict
 
     $failureCount = 0
+    $baselineMatchCount = 0
+    $regressedVsBase = 0
+    $unattributedFailures = 0
     $platforms = @()
     if (Test-Path $ContextJsonPath) {
         try {
             $context = Get-Content -Path $ContextJsonPath -Raw -Encoding UTF8 | ConvertFrom-Json
             $failureCount = @($context.failures.unique).Count
+            $baselineMatchCount = [int]$context.failures.baselineMatchCount
+            $regressedVsBase = [int]$context.gate.legsRegressedVsBase
+            $unattributedFailures = [int]$context.gate.unattributedFailures
             $platforms = @($context.failures.unique | ForEach-Object { $_.platform } | Where-Object { $_ -and $_ -ne "unknown" } | Select-Object -Unique)
         }
         catch {
@@ -245,6 +256,17 @@ function New-TestFailureReviewBody {
     $badgeLines = @()
     $badgeLines += New-Badge -Label "Overall" -Message $verdict -Color $verdictColor -Alt "Overall $verdict"
     $badgeLines += New-Badge -Label "Failures" -Message "$failureCount" -Color "8250df" -Alt "Failures $failureCount"
+    $badgeLines += New-Badge -Label "Baseline" -Message "$baselineMatchCount on base" -Color "0969da" -Alt "Baseline $baselineMatchCount on base"
+    # Surface the deterministic job-level regression count (red on PR, green on base) when
+    # any leg regressed -- it is the strongest PR-caused signal and caps the verdict ceiling.
+    if ($regressedVsBase -gt 0) {
+        $badgeLines += New-Badge -Label "Regressed" -Message "$regressedVsBase vs base" -Color "d1242f" -Alt "Regressed $regressedVsBase vs base"
+    }
+    # Surface failures the deterministic prior could not attribute either way -- they cap the
+    # ceiling at "Needs human investigation" (neither dismissible nor provably PR-caused).
+    if ($unattributedFailures -gt 0) {
+        $badgeLines += New-Badge -Label "Unattributed" -Message "$unattributedFailures" -Color "bf8700" -Alt "Unattributed $unattributedFailures"
+    }
     foreach ($platform in $platforms) {
         $badgeLines += New-Badge -Label "Platform" -Message $platform -Color "0969da" -Alt "Platform $platform"
     }

.github/skills/azdo-build-investigator/SKILL.md

@@ -1,106 +1,68 @@
 ---
 name: azdo-build-investigator
-description: "Investigate CI failures for dotnet/maui PRs — build errors, Helix test logs, and binlog analysis. Use when asked about failing checks, CI status, test failures, 'why is CI red', 'build failed', 'what's failing on PR', Helix failures, or device test failures."
+description: "Investigate CI failures for dotnet/maui PRs — build errors, Helix test logs, and binlog analysis. Use when asked about failing checks, CI status, test failures, 'why is CI red', 'build failed', 'what's failing on PR', 'is this PR ready to merge', Helix failures, or device test failures."
 metadata:
   author: dotnet-maui
-  version: "2.0"
+  version: "3.0"
 ---
 
 # dotnet/maui CI Investigation Context
 
-This skill provides MAUI-specific context for CI investigation. Use it together with the `ci-analysis` skill (loaded from the `dotnet-dnceng@dotnet-arcade-skills` plugin via `.github/copilot/settings.json`).
-
-> **First**: invoke the `ci-analysis` skill — it handles the core investigation workflow using `Get-CIStatus.ps1` and `gh` CLI (with MCP tools as optional enhancements if available). This skill provides MAUI-specific corrections and context on top of that.
-
-## Script Location
-
-The `ci-analysis` skill and its `Get-CIStatus.ps1` script are loaded automatically from the `dotnet/arcade-skills` plugin (configured in `.github/copilot/settings.json` via `enabledPlugins`). The CLI caches scripts to `~/.copilot/installed-plugins/dotnet-arcade-skills/`. No manual download is needed.
-
-## MAUI CI Pipelines
-
-> ⚠️ The `ci-analysis` skill's reference doc lists `maui-public` as the MAUI pipeline — **this is outdated**. The correct pipeline names are below.
-
-| Pipeline Name | Definition ID | Purpose |
-|---------------|---------------|---------|
-| `maui-pr` | **302** | Main build — check this first |
-| `maui-pr-devicetests` | **314** | Helix device tests (iOS, Android, Windows, MacCatalyst) |
-| `maui-pr-uitests` | **313** | Appium-based UI tests |
-
-**Organization**: `dnceng-public` / project `public`
-
-**Investigation priority order**: `maui-pr` → `maui-pr-devicetests` → `maui-pr-uitests`
-
-Most failures are in `maui-pr`. Device test failures appear in `maui-pr-devicetests`. Focus on the first failing pipeline before checking others.
-
-**When CI hasn't run:** Community PRs require a maintainer to trigger builds. Use `/azp run maui-pr` (or `maui-pr-devicetests`, `maui-pr-uitests`) in a PR comment, or trigger via Azure CLI. Not all pipelines run automatically — `maui-pr-devicetests` and `maui-pr-uitests` may need explicit triggers depending on the changed files.
-
-**Escalation:** For deep Helix log analysis (recurring failures, machine-specific issues, comparing passing vs. failing runs), escalate to the `helix-investigation` skill.
-
-## MAUI-Specific Quirks
-
-### XHarness Exit-0 Blind Spot
-
-XHarness (used for iOS/Android device tests in `maui-pr-devicetests`) **exits with code 0 even when tests fail**. This means:
-- The ADO job shows ✅ "Succeeded"
-- `ci-analysis` may report no failures
-- But actual test failures are hidden inside the Helix work items
-
-**How to detect hidden test failures**: Query the `ResultSummaryByBuild` Helix API endpoint:
-```
-GET https://helix.dot.net/api/2019-06-17/jobs/{correlationId}/aggregated
-```
-Look for `Failed` > 0 in the response even when the ADO build job shows green.
-
-When `ci-analysis` reports a `maui-pr-devicetests` build as passing but the PR has a `s/agent-gate-failed` label or the user suspects device test failures, always cross-check Helix `ResultSummaryByBuild`.
-
-### Container Artifact Binlogs
-
-MAUI build artifacts are **Container type**, not `PipelineArtifact`. This means:
-- `az pipelines runs artifact download` does **not** work for binlogs
-- Artifact names are like `Windows_NT_Build Windows (Debug)_Attempt1` (not `binlog`)
-- Download requires a Bearer token from `az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798`
-- Use the ADO File Container API: `/_apis/resources/Containers/{id}?api-version=5.0-preview&$format=OctetStream`
-
-If available, use the `mcp-binlog-tool` MCP server to analyze downloaded `.binlog` files. This is optional — the core investigation workflow works without it via `gh` CLI and REST APIs.
-
-## Common MAUI Failure Patterns
-
-| Pattern | Where | Notes |
-|---------|-------|-------|
-| `error CS####` | `maui-pr` | C# compiler error — check file/line |
-| `error XA####` | `maui-pr` | Android build error |
-| `XamlC` | `maui-pr` | XAML compiler — usually missing type or bad binding |
-| `error XAGRDL0000` / `401` / `No local versions` | `maui-pr` or official build | Gradle/Maven feed issue — see below |
-| `XHarness timeout` | `maui-pr-devicetests` Helix logs | Test killed by infrastructure; may be transient |
-| `No test result files found` | `maui-pr-devicetests` Helix logs | Tests never ran or app crashed on launch |
-| UI test screenshot diff | `maui-pr-uitests` | Visual regression; check baseline images |
-
-## Test Count Deduplication
-
-When querying AzDO test results directly (e.g., via the `/test/runs/{id}/results` API), **always deduplicate before reporting counts**. MAUI UI tests produce multiple test runs per test because each test executes across:
-- **Runtime variants**: CoreCLR and Mono
-- **Platform versions**: e.g., iOS 18.5 and iOS latest, Android API 30 and API 36
-- **Retry attempts**: failed jobs are retried, each attempt publishes a new test run
-
-A single failing test can appear in 4–8+ test runs. Summing raw `totalTests - passedTests` across all runs inflates failure counts dramatically.
-
-**How to deduplicate**: Group by **test name + OS platform** (extract the OS token — `ios`, `android`, `mac`, `win` — from the run name as the grouping key). For example, "DatePicker_Format_D on iOS" vs "DatePicker_Format_D on Android" are distinct failures worth reporting separately. Collapse retries and runtime variants (coreclr/mono) of the same test on the same OS — if a test fails on both coreclr and mono for iOS, that's one issue, not two.
-
-### Gradle / Maven / CFSClean Failures
-
-**Error signatures:**
-```
-error XAGRDL0000: Could not resolve com.android.tools.build:gradle:8.11.1
-  > Received status code 401: Unauthorized - No local versions of package
-```
-```
-error XAGRDL0000: Could not GET '...pkgs.dev.azure.com/.../maven/v1/...'
-  > Unauthorized - Please provide authentication to save package from upstream
-```
-
-**Fix:** Tell the user to run `./eng/ingest-maven-deps.sh` locally to pre-ingest packages into the feed.
-
-**Do NOT:**
-- Remove CFSClean from `ci-official.yml` — security compliance requirement
-- Upgrade Gradle past 8.x — `dotnet/android#10738`
-- Add `mavenCentral()` or `google()` back — use the Azure Artifacts feed
+This skill provides MAUI-specific context for **interactive** CI investigation —
+answering "why is CI red?" and "is this PR ready to merge?". Use it together with the
+`ci-analysis` skill (loaded from the `dotnet-dnceng@dotnet-arcade-skills` plugin via
+`.github/copilot/settings.json`).
+
+> **First**: invoke the `ci-analysis` skill — it handles the core investigation workflow
+> using `Get-CIStatus.ps1` and `gh` CLI (with MCP tools as optional enhancements if
+> available). This skill provides MAUI-specific corrections and context on top of that.
+
+For the **automated** path (a maintainer comments `/review tests` on a PR), the
+`review-test-failures` skill does the same classification deterministically and posts a
+merge-readiness comment. Both skills reason from the same shared facts below.
+
+## MAUI CI facts (canonical)
+
+All MAUI-specific facts live in **`.github/docs/maui-ci-facts.md`** — read it. It covers:
+
+- **Pipeline names + definition IDs** (`maui-pr` 302, `maui-pr-devicetests` 314,
+  `maui-pr-uitests` 313), org/project, and investigation priority order.
+- **AzDO data sources** (anonymous public build/timeline/log REST APIs; `_apis/test` is
+  optional enrichment).
+- **XHarness exit-0 blind spot** and the Helix `aggregated` endpoint for hidden
+  device-test failures.
+- **Container artifact binlogs** (Bearer token + File Container API; `mcp-binlog-tool`).
+- **Test count deduplication** (group by test name + OS platform).
+- **Baseline comparison** (is the failure already red on the base branch?).
+- **Visual baseline** and **platform mismatch** guidance.
+- **Gradle / Maven / CFSClean** signatures and the `./eng/ingest-maven-deps.sh` fix.
+- **Common failure patterns** table.
+- **Merge-readiness criteria** — the shared definition of `Ready to merge` / `Not ready`
+  / `Needs human investigation` / `Insufficient data` / `No failures found`.
+
+Do not restate those facts from memory; change them in the canonical doc only.
+
+## Script location
+
+The `ci-analysis` skill and its `Get-CIStatus.ps1` script are loaded automatically from
+the `dotnet/arcade-skills` plugin (configured in `.github/copilot/settings.json` via
+`enabledPlugins`). The CLI caches scripts to
+`~/.copilot/installed-plugins/dotnet-arcade-skills/`. No manual download is needed.
+
+> ⚠️ The `ci-analysis` skill's reference doc lists `maui-public` as the MAUI pipeline —
+> **this is outdated**. Use the pipeline names in `.github/docs/maui-ci-facts.md`.
+
+## Using this skill
+
+1. Invoke `ci-analysis` to gather build/test status for the PR.
+2. Apply the MAUI corrections from `.github/docs/maui-ci-facts.md` — especially the
+   correct pipeline names, the XHarness exit-0 cross-check on `maui-pr-devicetests`, and
+   test-count deduplication.
+3. When asked whether a PR is ready to merge, compare failures against the base branch
+   and apply the merge-readiness criteria from the canonical doc.
+4. **Escalation:** for deep Helix log analysis (recurring failures, machine-specific
+   issues, comparing passing vs. failing runs), escalate to the `helix-investigation`
+   skill.
+
+**When CI hasn't run:** community PRs require a maintainer to trigger builds — see the
+canonical doc.