diff --git a/.github/actions/scrape-and-improve/action.yml b/.github/actions/scrape-and-improve/action.yml new file mode 100644 index 000000000000..f975a4e6673e --- /dev/null +++ b/.github/actions/scrape-and-improve/action.yml @@ -0,0 +1,110 @@ +name: 'Scrape and Improve' +description: 'Collects agent interaction data and generates instruction improvement recommendations.' + +inputs: + token: + description: 'GitHub token for API access' + required: false + default: ${{ github.token }} + pr-numbers: + description: 'Comma-separated list of PR numbers to analyze (optional)' + required: false + default: '' + label: + description: 'GitHub label to filter PRs by' + required: false + default: 'copilot' + since: + description: 'ISO 8601 date to filter PRs from (e.g., 2026-01-01)' + required: false + default: '' + recent-pr-count: + description: 'Number of most recent PRs to scrape for suggestion analysis' + required: false + default: '20' + memory-context: + description: 'Raw memory context text from agent interactions to analyze' + required: false + default: '' + +outputs: + report-file: + description: 'Path to the analysis report' + value: ${{ steps.analyze.outputs.report-file }} + recommendations-file: + description: 'Path to the recommendations JSON' + value: ${{ steps.analyze.outputs.recommendations-file }} + recommendations-count: + description: 'Number of recommendations generated' + value: ${{ steps.analyze.outputs.recommendations-count }} + +runs: + using: 'composite' + steps: + + - name: Collect agent data + id: collect + shell: pwsh + env: + GH_TOKEN: ${{ inputs.token }} + run: | + Write-Host "Collecting agent interaction data..." + $params = @{ + RepoRoot = "${{ github.workspace }}" + OutputDir = "${{ runner.temp }}/scrape-and-improve" + Repository = "${{ github.repository }}" + RecentPRCount = ${{ inputs.recent-pr-count }} + } + if ("${{ inputs.pr-numbers }}") { + $params["PRNumbers"] = "${{ inputs.pr-numbers }}" + } + if ("${{ inputs.label }}") { + $params["Label"] = "${{ inputs.label }}" + } + if ("${{ inputs.since }}") { + $params["Since"] = "${{ inputs.since }}" + } + if ("${{ inputs.memory-context }}") { + $params["MemoryContext"] = "${{ inputs.memory-context }}" + } + & "${{ github.workspace }}/.github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1" @params + + - name: Analyze and generate recommendations + id: analyze + shell: pwsh + run: | + Write-Host "Analyzing collected data..." + $outputDir = "${{ runner.temp }}/scrape-and-improve" + & "${{ github.workspace }}/.github/skills/scrape-and-improve/scripts/Analyze-And-Recommend.ps1" ` + -InputFile "$outputDir/collected-data.json" ` + -OutputDir $outputDir ` + -RepoRoot "${{ github.workspace }}" + + "report-file=$outputDir/analysis-report.md" >> $env:GITHUB_OUTPUT + "recommendations-file=$outputDir/recommendations.json" >> $env:GITHUB_OUTPUT + + $recommendations = Get-Content -Path "$outputDir/recommendations.json" -Raw | ConvertFrom-Json + $count = if ($recommendations -is [System.Array]) { $recommendations.Count } else { if ($recommendations) { 1 } else { 0 } } + "recommendations-count=$count" >> $env:GITHUB_OUTPUT + + - name: Upload analysis artifacts + uses: actions/upload-artifact@v4 + with: + name: scrape-and-improve-results + path: | + ${{ runner.temp }}/scrape-and-improve/analysis-report.md + ${{ runner.temp }}/scrape-and-improve/recommendations.json + ${{ runner.temp }}/scrape-and-improve/collected-data.json + retention-days: 30 + + - name: Print summary + shell: pwsh + run: | + $reportFile = "${{ steps.analyze.outputs.report-file }}" + if (Test-Path $reportFile) { + Write-Host "::group::Analysis Report" + Get-Content -Path $reportFile + Write-Host "::endgroup::" + } + Write-Host "" + Write-Host "Recommendations: ${{ steps.analyze.outputs.recommendations-count }}" diff --git a/.github/agents/scrape-and-improve.md b/.github/agents/scrape-and-improve.md new file mode 100644 index 000000000000..213753b513ce --- /dev/null +++ b/.github/agents/scrape-and-improve.md @@ -0,0 +1,167 @@ +--- +name: scrape-and-improve +description: Scrapes agent PR sessions, Copilot comments, CCA session data, and repository memories to identify patterns and generate instruction file updates for improved agent success rates. +--- + +# Scrape and Improve Agent + +Autonomously collects data from multiple agent interaction sources, analyzes patterns of success and failure, and **applies** instruction file updates to improve future agent performance. + +## When to Invoke + +- "Scrape and improve agent instructions" +- "Analyze agent patterns and apply improvements" +- "What patterns are agents struggling with? Fix them." +- "Run scrape-and-improve and check in improvements" +- Periodically (weekly/monthly) to improve agent instructions + +## When NOT to Invoke + +- For a single PR analysis → Use `learn-from-pr` agent or `/learn-from-pr` skill +- During active PR work → Use `pr` agent +- For analysis only without applying changes → Use `/scrape-and-improve` skill directly + +--- + +## Workflow + +### Phase 1: Collect Data + +Run the data collection script to gather information from all sources: + +```bash +pwsh .github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 \ + -RepoRoot "." \ + -RecentPRCount 20 +``` + +**Data sources collected:** + +| Source | Location | What It Contains | +|--------|----------|------------------| +| Agent PR Sessions | `.github/agent-pr-session/*.md` | Fix candidates, root causes, phase statuses | +| Copilot Comments | PR comments via `gh` CLI | AI Summary, try-fix attempts, test verification | +| CCA Session Logs | `CustomAgentLogsTmp/PRState/` | Session state, fix attempts, convention patterns | +| Repository Memories | Agent memory context | Stored facts (conventions, build commands, patterns) | +| Recent PR Reviews | Most recent 20 PRs via `gh` API | Review comments, suggestion acceptance/rejection | + +If the agent has access to repository memories (provided in the conversation context), pass them via `-MemoryContext`: + +```bash +pwsh .github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 \ + -RepoRoot "." \ + -RecentPRCount 20 \ + -MemoryContext "" +``` + +### Phase 2: Analyze Patterns + +Run the analysis script to identify patterns across collected data: + +```bash +pwsh .github/skills/scrape-and-improve/scripts/Analyze-And-Recommend.ps1 \ + -RepoRoot "." +``` + +This produces: +- `CustomAgentLogsTmp/scrape-and-improve/analysis-report.md` — Full report +- `CustomAgentLogsTmp/scrape-and-improve/recommendations.json` — Structured recommendations + +### Phase 3: Present Results + +Display the analysis report to the user, including: + +1. **Data Summary** — What was collected (session files, comments, memories, PRs) +2. **Pattern Analysis** — Slow discovery PRs, quick successes, root causes +3. **Memory Analysis** — Subject frequency, convention patterns +4. **Recent PR Suggestion Analysis** — Acceptance/rejection rates, hotspot areas +5. **Recommendations** — Prioritized list with evidence and impact + +### Phase 4: Apply Improvements + +For each **High or Medium priority** recommendation from the analysis (priorities are determined by the `Analyze-And-Recommend.ps1` script based on evidence strength, recurrence across PRs, and expected impact on agent success rates): + +| Category | Action | +|----------|--------| +| Instruction file | Edit existing or create new `.github/instructions/*.instructions.md` | +| Skill enhancement | Edit `.github/skills/*/SKILL.md` | +| Architecture doc | Edit `/docs/design/*.md` or create in `.github/architecture/` | +| General AI guidance | Edit `.github/copilot-instructions.md` | + +**Before each edit:** +- Read the target file first +- Check for existing similar content (don't duplicate) +- Match the existing style/format +- Find the appropriate section + +**Skip applying if:** +- Content already exists +- Recommendation is too vague +- Would require major restructuring + +### Phase 5: Verify and Report + +After applying changes: + +1. Run `git diff` to review all edits +2. Verify no syntax errors in modified files (valid markdown) +3. Confirm style matches existing content + +Present a summary: + +```markdown +## Scrape and Improve Results + +### Data Collected +| Source | Count | +|--------|-------| +| [source] | [count] | + +### Recommendations Applied +| File | Change | +|------|--------| +| [path] | [what was added/modified] | + +### Recommendations Not Applied +| Recommendation | Reason | +|----------------|--------| +| [rec] | [why skipped] | + +### Key Findings +- [finding 1] +- [finding 2] +``` + +--- + +## Error Handling + +| Situation | Action | +|-----------|--------| +| No agent PR sessions found | Report empty, suggest checking date range | +| `gh` CLI not authenticated | Report error, proceed with local data only | +| No Copilot comments found | Proceed with session files and memories | +| No CCA session data | Proceed with other sources | +| No patterns identified | Report "insufficient data" with data source summary | +| Target file doesn't exist | Create if instruction doc, skip if code | +| Duplicate content exists | Skip, note in report | + +## Constraints + +- **Evidence-based** — Every recommendation must cite specific PRs or data +- **Don't duplicate** — Check existing instruction files before recommending or applying +- **Match style** — Read file before editing +- **Only apply High/Medium priority** — Report Low priority without applying +- **Focus on actionable** — Skip vague observations +- **Respect existing patterns** — Match style and structure of existing instruction files + +--- + +## Difference from Skill + +| Aspect | `/scrape-and-improve` Skill | This Agent | +|--------|----------------------------|------------| +| Output | Analysis report and recommendations | Applied changes to instruction files | +| Mode | Analysis only | Autonomous — collects, analyzes, applies | +| Use when | Want to review without applying | Want full automation | +| Trigger | "analyze agent patterns" | "scrape and improve, check in improvements" | diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 5c9f96eba10f..f8da373dba04 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -78,6 +78,39 @@ Platform-specific files use naming conventions to control compilation: - `src/Essentials/samples/Essentials.Sample` - Essentials API demonstrations (non-UI MAUI APIs) - `src/BlazorWebView/samples/` - BlazorWebView sample applications +## Common Fix Patterns (From Agent Analysis) + +These patterns were identified from scrape-and-improve analysis of past agent PR sessions. They help agents find fixes faster by documenting what worked and what didn't. + +### NavigationPage Handler Disconnection + +When fixing handler disconnection issues with `NavigationPage.RemovePage()`: +- ❌ **Don't** add `DisconnectHandlers()` inside `SendHandlerUpdateAsync` callback (failed in PR #32289) +- ❌ **Don't** add `DisconnectHandlers()` synchronously after `SendHandlerUpdateAsync` (failed in PR #32289) +- ✅ **Do** add `DisconnectHandlers()` at the end of the Legacy `RemovePage` method path +- **Key insight**: The fix location is in the legacy code path, not the modern handler update path + +### CollectionView EmptyView (Android) + +When fixing EmptyView display issues inside unconstrained layouts (e.g., `VerticalStackLayout`): +- ❌ **Don't** just check for infinity before casting in `GetHeight()`/`GetWidth()` (failed in PR #33134) +- ✅ **Do** handle `double.PositiveInfinity` → `int.MaxValue` propagation in `EmptyViewAdapter` +- **Key insight**: In C#, `(int)double.PositiveInfinity` produces `int.MaxValue`. The fix must normalize dimensions where they're consumed (e.g., convert `int.MaxValue` back to `double.PositiveInfinity` for layout measurement), not where they're cast +- **Root cause**: When CollectionView is inside VerticalStackLayout, height constraint is `double.PositiveInfinity` which propagates through `EmptyViewAdapter.RecyclerViewHeight` + +### Shell Navigation Tests (iOS) + +When fixing Shell back button / navigation tests on iOS: +- Check if `OnAppearing` count expectations changed due to lifecycle changes in related PRs +- Shell navigation involves both `DidPopItem` and `SendPop()` — ensure stacks stay in sync +- **Key insight**: Simplify `DidPopItem` to always call `SendPop()` when stacks are out of sync + +### Device Test Isolation + +When fixing flaky device tests (e.g., Stepper tests): +- Tests that modify control state (like `Minimum`, `Maximum`) can leak state to subsequent tests +- **Key insight**: Check if a previous test in the same class modifies shared state without resetting it + ## Development Workflow ### Testing @@ -220,6 +253,13 @@ The repository includes specialized custom agents and reusable skills for specif - **Output**: Applied changes to instruction files, skills, architecture docs, code comments - **Do NOT use for**: Analysis only without applying changes → Use `/learn-from-pr` skill instead +5. **scrape-and-improve** - Scrapes agent data from multiple sources and applies instruction improvements + - **Use when**: Want to analyze agent patterns across multiple PRs and apply improvements automatically + - **Capabilities**: Collects data from PR sessions, Copilot comments, CCA sessions, memories, and recent PRs; analyzes patterns; applies High/Medium priority instruction updates + - **Trigger phrases**: "scrape and improve agent instructions", "analyze agent patterns and apply improvements", "run scrape-and-improve and check in improvements" + - **Output**: Applied changes to instruction files based on aggregated pattern analysis + - **Do NOT use for**: Analysis only without applying → Use `/scrape-and-improve` skill instead; single PR analysis → Use `learn-from-pr` agent + ### Reusable Skills Skills are modular capabilities that can be invoked directly or used by agents. Located in `.github/skills/`: @@ -277,9 +317,15 @@ Skills are modular capabilities that can be invoked directly or used by agents. - **Categories**: Build, WindowsTemplates, macOSTemplates, Blazor, MultiProject, Samples, AOT, RunOnAndroid, RunOniOS - **Note**: **ALWAYS use this skill** instead of manual `dotnet test` commands for integration tests +9. **scrape-and-improve** (`.github/skills/scrape-and-improve/SKILL.md`) + - **Purpose**: Scrapes agent PR sessions, Copilot comments, CCA sessions, and memories to generate instruction updates + - **Trigger phrases**: "improve agent instructions", "scrape agent data", "analyze agent patterns", "what patterns are agents struggling with?" + - **Scripts**: `Collect-AgentData.ps1`, `Analyze-And-Recommend.ps1` + - **Note**: Also available as a GitHub Action (`.github/actions/scrape-and-improve/`) and scheduled workflow + #### Internal Skills (Used by Agents) -9. **try-fix** (`.github/skills/try-fix/SKILL.md`) +10. **try-fix** (`.github/skills/try-fix/SKILL.md`) - **Purpose**: Proposes ONE independent fix approach, applies it, tests, records result with failure analysis, then reverts - **Used by**: pr agent Phase 3 (Fix phase) - rarely invoked directly by users - **Behavior**: Reads prior attempts to learn from failures. Max 5 attempts per session. diff --git a/.github/skills/scrape-and-improve/SKILL.md b/.github/skills/scrape-and-improve/SKILL.md new file mode 100644 index 000000000000..bf037823eb61 --- /dev/null +++ b/.github/skills/scrape-and-improve/SKILL.md @@ -0,0 +1,232 @@ +--- +name: scrape-and-improve +description: Scrapes agent PR sessions, Copilot comments, CCA session data, and repository memories to identify patterns and generate instruction file updates for improved agent success rates. +metadata: + author: dotnet-maui + version: "1.0" +compatibility: Requires GitHub CLI (gh) authenticated with access to dotnet/maui repository. +--- + +# Scrape and Improve + +Collects data from multiple agent interaction sources, analyzes patterns of success and failure, and generates actionable instruction file updates to improve future agent performance. + +## Data Sources + +| Source | Location | What It Contains | +|--------|----------|------------------| +| Agent PR Sessions | `.github/agent-pr-session/*.md` | Structured PR review records with phases, fix candidates, root cause analysis | +| Copilot Comments | PR comments via `gh` CLI | AI Summary comments, try-fix attempts, test verification results | +| CCA Session Logs | `CustomAgentLogsTmp/PRState/` | Live session state files from Copilot Coding Agent runs | +| Repository Memories | Agent memory context | Stored facts from agent interactions (conventions, build commands, patterns) | +| Recent PR Reviews | Most recent N PRs via `gh` API | Review comments, suggestion acceptance/rejection, code change requests | +| Copilot Review Comments | PR review comments via `gh` CLI | Code review feedback, inline suggestions | + +## Inputs + +| Input | Required | Source | +|-------|----------|--------| +| Date range | No | Default: last 30 days | +| PR numbers | No | Specific PRs to analyze (comma-separated) | +| Label filter | No | Filter PRs by label (e.g., `copilot`) | +| Recent PR count | No | Number of recent PRs to scrape (default: 20) | +| Memory context | No | Raw memory text from agent interactions | + +## Outputs + +1. **Data Collection Report** - Summary of gathered data: + - Number of PR sessions analyzed + - Number of Copilot comments scraped + - Number of CCA sessions found + - Key patterns identified + +2. **Improvement Recommendations** - Prioritized list of instruction updates: + - Category, Priority, Location, Specific Change, Evidence + +3. **Generated Instruction Updates** - Ready-to-apply changes: + - Diffs for existing instruction files + - New instruction file proposals + +## Completion Criteria + +The skill is complete when you have: +- [ ] Collected data from all available sources +- [ ] Identified recurring patterns (failures, slow successes, quick successes) +- [ ] Generated at least one concrete instruction update +- [ ] Presented findings with evidence from specific PRs + +## When to Use + +- Periodically (weekly/monthly) to improve agent instructions +- After a batch of agent-assisted PRs +- When agent success rates seem low +- When asked "improve agent instructions" or "what patterns are agents struggling with?" + +## When NOT to Use + +- For a single PR analysis → Use `learn-from-pr` skill instead +- During active PR work → Use `pr` agent instead +- For writing tests → Use `write-tests-agent` instead + +--- + +## Workflow + +### Step 1: Collect Data + +Run the data collection script to gather information from all sources: + +```bash +# Collect from all sources (last 30 days) +pwsh .github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 + +# Collect for specific PRs +pwsh .github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 -PRNumbers "33380,33134,33392" + +# Collect PRs with specific label +pwsh .github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 -Label "copilot" + +# Collect from a specific date range +pwsh .github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 -Since "2026-01-01" + +# Scrape most recent 20 PRs for suggestion response analysis +pwsh .github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 -RecentPRCount 20 + +# Pass memory context for analysis +pwsh .github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 -MemoryContext "**subject**\n- Fact: ...\n- Citations: ..." +``` + +The script collects: + +1. **Agent PR Session files** from `.github/agent-pr-session/`: + - Parses fix candidates tables (approach, result, files changed) + - Extracts root cause analysis sections + - Identifies failure patterns and attempt counts + +2. **Copilot comments on PRs**: + - Finds PRs with `copilot` label or AI Summary comments + - Extracts try-fix attempt results + - Collects test verification outcomes + - Gathers code review findings + +3. **CCA session data** from `CustomAgentLogsTmp/PRState/`: + - Reads session state files + - Extracts fix attempt records + - Identifies files targeted vs actual fix location + +4. **Repository memories** (stored agent facts): + - Parses structured memory blocks (subject, fact, citations) + - Scans session files for convention/build-command patterns + - Categorizes memories by subject for frequency analysis + +5. **Recent PR suggestion responses** (most recent N PRs): + - Fetches review comments from recent PRs via GitHub API + - Detects Copilot-authored suggestions + - Tracks acceptance/rejection/discussion patterns + - Identifies review hotspot areas by file path + - Analyzes code change request patterns + +Output is saved to `CustomAgentLogsTmp/scrape-and-improve/collected-data.json`. + +### Step 2: Analyze Patterns + +Run the analysis script to identify patterns across collected data: + +```bash +# Analyze collected data and generate recommendations +pwsh .github/skills/scrape-and-improve/scripts/Analyze-And-Recommend.ps1 + +# Analyze with specific input file +pwsh .github/skills/scrape-and-improve/scripts/Analyze-And-Recommend.ps1 -InputFile CustomAgentLogsTmp/scrape-and-improve/collected-data.json +``` + +The script analyzes: + +| Pattern Category | What It Looks For | Example Finding | +|-----------------|-------------------|-----------------| +| **Wrong File** | Fix attempts targeting different files than final fix | "3/5 PRs had agents looking in handler when fix was in core" | +| **Slow Discovery** | PRs with >3 fix attempts before success | "Shell navigation fixes average 4.2 attempts" | +| **Quick Success** | PRs with 1 fix attempt | "Button fixes succeed first try when searching for property pattern" | +| **Platform Gaps** | Platform-specific failures not covered by instructions | "No iOS instruction for TraitCollection lifecycle" | +| **Test Gaps** | Missing test patterns for common scenarios | "CollectionView empty state tests missing" | +| **Common Root Causes** | Frequently appearing root causes | "int/double casting issues in Android layout code" | +| **Memory Hotspots** | Frequently stored memory subjects | "convention patterns stored 5 times - should be in instructions" | +| **Suggestion Rejection** | High rejection rate on Copilot suggestions | "40% rejection rate in Controls area - needs better guidance" | +| **Review Hotspots** | Areas with high review comment volume | "Core/ has 15 review comments across 20 PRs" | + +Output is saved to `CustomAgentLogsTmp/scrape-and-improve/analysis-report.md`. + +### Step 3: Generate Instruction Updates + +Based on the analysis, generate concrete instruction file changes: + +The analysis script produces: + +1. **Updates to existing instruction files** - Specific additions to `.github/instructions/*.instructions.md` +2. **New instruction file proposals** - When a pattern doesn't fit existing files +3. **Skill enhancements** - Updates to `.github/skills/*/SKILL.md` +4. **copilot-instructions.md updates** - General guidance additions + +Each recommendation includes: +- **Evidence**: Which PRs demonstrated this pattern +- **Priority**: High (prevents class of bugs) / Medium (helps discovery) / Low (nice to have) +- **Location**: Exact file path and section +- **Specific Change**: The text to add or modify +- **Impact**: Expected improvement in agent success rate + +### Step 4: Review and Apply + +Present the recommendations to the user for review. The user can: + +1. **Apply all** - Accept all generated changes +2. **Apply selectively** - Choose which changes to apply +3. **Modify and apply** - Adjust recommendations before applying +4. **Save for later** - Keep the report without applying + +When applying, use the learn-from-pr agent's Phase 2 approach: +- Read target file first +- Check for existing similar content +- Match existing style/format +- Find appropriate section + +--- + +## Pattern-to-Improvement Mapping + +| Pattern | Improvement Type | Target | +|---------|-----------------|--------| +| Wrong file entirely | Architecture doc | `.github/instructions/` - component relationships | +| Tunnel vision on error message | Instruction file | "Search for PATTERN across codebase" | +| Missing platform knowledge | Platform instruction | `.github/instructions/{platform}.instructions.md` | +| Wrong abstraction layer | Architecture doc | Handler vs core layer guidance | +| Repeated root cause type | Code comment | Source file with non-obvious behavior | +| Slow test discovery | Skill enhancement | try-fix SKILL.md search strategies | +| Quick success pattern | Skill reinforcement | Document what worked and why | + +--- + +## Error Handling + +| Situation | Action | +|-----------|--------| +| No agent PR sessions found | Report empty, suggest checking date range | +| `gh` CLI not authenticated | Report error with authentication instructions | +| No Copilot comments found | Proceed with session files only | +| No CCA session data | Proceed with PR sessions and comments only | +| No patterns identified | Report "insufficient data" with data source summary | + +## Constraints + +- **Read-only analysis** - Don't apply changes without user approval (unless triggered by CI action) +- **Evidence-based** - Every recommendation must cite specific PRs +- **Don't duplicate** - Check existing instruction files before recommending +- **Focus on actionable** - Skip vague observations, only recommend specific changes +- **Respect existing patterns** - Match style and structure of existing instruction files + +--- + +## Integration + +- **learn-from-pr** → Single PR analysis (this skill aggregates across many PRs) +- **pr agent** → Uses instruction files that this skill improves +- **GitHub Action** → Can trigger this skill automatically on schedule or manually diff --git a/.github/skills/scrape-and-improve/scripts/Analyze-And-Recommend.ps1 b/.github/skills/scrape-and-improve/scripts/Analyze-And-Recommend.ps1 new file mode 100644 index 000000000000..3bd646c3d02c --- /dev/null +++ b/.github/skills/scrape-and-improve/scripts/Analyze-And-Recommend.ps1 @@ -0,0 +1,544 @@ +<# +.SYNOPSIS + Analyzes collected agent data and generates instruction improvement recommendations. + +.DESCRIPTION + Reads the collected data from Collect-AgentData.ps1 and identifies patterns + across agent interactions, memories, and PR review responses. Produces a + markdown report with prioritized recommendations for instruction file updates. + +.PARAMETER InputFile + Path to the collected data JSON file. Default: CustomAgentLogsTmp/scrape-and-improve/collected-data.json + +.PARAMETER OutputDir + Output directory for the analysis report. Default: CustomAgentLogsTmp/scrape-and-improve + +.PARAMETER RepoRoot + Repository root directory. Default: current directory. + +.EXAMPLE + pwsh .github/skills/scrape-and-improve/scripts/Analyze-And-Recommend.ps1 + pwsh .github/skills/scrape-and-improve/scripts/Analyze-And-Recommend.ps1 -InputFile path/to/data.json +#> + +param( + [string]$InputFile = "CustomAgentLogsTmp/scrape-and-improve/collected-data.json", + [string]$OutputDir = "CustomAgentLogsTmp/scrape-and-improve", + [string]$RepoRoot = "." +) + +$ErrorActionPreference = "Continue" + +# Analysis thresholds +$HIGH_REJECTION_THRESHOLD = 30 # Percentage - flag when Copilot suggestion rejection rate exceeds this +$HOTSPOT_COMMENT_THRESHOLD = 5 # Count - flag areas with this many or more review comments +$MEMORY_FREQUENCY_THRESHOLD = 2 # Count - flag memory subjects appearing this many or more times + +# Ensure output directory exists +New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host " Scrape and Improve: Analysis" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan + +# ───────────────────────────────────────────────────────────── +# Load collected data +# ───────────────────────────────────────────────────────────── +if (-not (Test-Path $InputFile)) { + Write-Host "❌ Input file not found: $InputFile" -ForegroundColor Red + Write-Host " Run Collect-AgentData.ps1 first." -ForegroundColor Red + exit 1 +} + +$data = Get-Content -Path $InputFile -Raw | ConvertFrom-Json + +Write-Host "`n📊 Loaded data from $InputFile" +Write-Host " PRs analyzed: $($data.summary.totalPRsAnalyzed)" +Write-Host " Fix attempts: $($data.summary.totalFixAttempts)" + +# ───────────────────────────────────────────────────────────── +# Identify existing instruction files +# ───────────────────────────────────────────────────────────── +Write-Host "`n📄 Scanning existing instruction files..." -ForegroundColor Yellow + +$instructionDir = Join-Path $RepoRoot ".github/instructions" +$existingInstructions = @() +if (Test-Path $instructionDir) { + $existingInstructions = Get-ChildItem -Path $instructionDir -Filter "*.instructions.md" -File | + ForEach-Object { $_.Name } + Write-Host " Found $($existingInstructions.Count) instruction file(s)" +} + +$skillDir = Join-Path $RepoRoot ".github/skills" +$existingSkills = @() +if (Test-Path $skillDir) { + $existingSkills = Get-ChildItem -Path $skillDir -Directory | + ForEach-Object { $_.Name } + Write-Host " Found $($existingSkills.Count) skill(s)" +} + +# ───────────────────────────────────────────────────────────── +# Pattern Analysis +# ───────────────────────────────────────────────────────────── +Write-Host "`n🔍 Analyzing patterns..." -ForegroundColor Yellow + +$patterns = @{ + multipleAttempts = @() # PRs with >2 fix attempts + singleAttempt = @() # PRs with exactly 1 attempt (quick success) + failedAttempts = @() # Individual failed fix attempts + successfulAttempts = @() # Individual successful fix attempts + rootCauses = @() # Extracted root causes + platformPatterns = @{} # Platform-specific patterns +} + +foreach ($session in $data.sources.agentSessions) { + $attemptCount = $session.fixCandidates.Count + + if ($attemptCount -gt 2) { + $patterns.multipleAttempts += @{ + pr = $session.prNumber + attempts = $attemptCount + file = $session.file + } + } elseif ($attemptCount -eq 1) { + $patterns.singleAttempt += @{ + pr = $session.prNumber + file = $session.file + } + } + + foreach ($fix in $session.fixCandidates) { + if ($fix.failed) { + $patterns.failedAttempts += @{ + pr = $session.prNumber + approach = $fix.approach + source = $fix.source + } + } + if ($fix.passed) { + $patterns.successfulAttempts += @{ + pr = $session.prNumber + approach = $fix.approach + source = $fix.source + } + } + } + + if ($session.rootCause) { + $patterns.rootCauses += @{ + pr = $session.prNumber + rootCause = $session.rootCause + } + } +} + +# Analyze Copilot comment patterns +$prsWithAISummary = ($data.sources.copilotComments | Where-Object { $_.aiSummaryFound }).Count +$prsWithTryFix = ($data.sources.copilotComments | Where-Object { $_.tryFixAttempts -gt 0 }).Count +$prsWithTestVerification = ($data.sources.copilotComments | Where-Object { $_.testVerification }).Count + +# ───────────────────────────────────────────────────────────── +# Analyze Memories +# ───────────────────────────────────────────────────────────── +Write-Host "`n🧠 Analyzing memories..." -ForegroundColor Yellow + +$memoryAnalysis = @{ + totalMemories = 0 + bySubject = @{} + bySource = @{} + conventions = @() + buildCommands = @() + storeEvents = @() +} + +if ($data.sources.memories) { + $memoryAnalysis.totalMemories = $data.sources.memories.Count + + foreach ($mem in $data.sources.memories) { + $subject = if ($mem.subject) { $mem.subject } else { "unknown" } + $source = if ($mem.source) { $mem.source } else { "unknown" } + + if (-not $memoryAnalysis.bySubject.ContainsKey($subject)) { + $memoryAnalysis.bySubject[$subject] = 0 + } + $memoryAnalysis.bySubject[$subject]++ + + if (-not $memoryAnalysis.bySource.ContainsKey($source)) { + $memoryAnalysis.bySource[$source] = 0 + } + $memoryAnalysis.bySource[$source]++ + + # Categorize + switch ($subject) { + "convention" { $memoryAnalysis.conventions += $mem } + "build-command" { $memoryAnalysis.buildCommands += $mem } + "store-event" { $memoryAnalysis.storeEvents += $mem } + } + } + Write-Host " Memories analyzed: $($memoryAnalysis.totalMemories)" + Write-Host " Subjects found: $($memoryAnalysis.bySubject.Count)" +} + +# ───────────────────────────────────────────────────────────── +# Analyze Recent PR Suggestion Responses +# ───────────────────────────────────────────────────────────── +Write-Host "`n📊 Analyzing recent PR suggestion responses..." -ForegroundColor Yellow + +$suggestionAnalysis = @{ + totalPRs = 0 + copilotPRs = 0 + totalReviewComments = 0 + totalCopilotSuggestions = 0 + acceptanceRate = 0 + rejectionRate = 0 + commonChangeRequests = @() + filePathPatterns = @{} + copilotPRList = @() +} + +if ($data.sources.recentPRSuggestions) { + $suggestionAnalysis.totalPRs = $data.sources.recentPRSuggestions.Count + + foreach ($prData in $data.sources.recentPRSuggestions) { + if ($prData.isCopilotPR) { + $suggestionAnalysis.copilotPRs++ + $suggestionAnalysis.copilotPRList += @{ + number = $prData.prNumber + title = $prData.title + state = $prData.state + } + } + + $stats = $prData.suggestionStats + $suggestionAnalysis.totalReviewComments += $stats.totalReviewComments + $suggestionAnalysis.totalCopilotSuggestions += $stats.copilotSuggestions + + # Track file path patterns in review comments + foreach ($rc in $prData.reviewComments) { + if ($rc.path) { + # Extract component area from path + $area = "" + if ($rc.path -match "src/Controls/") { $area = "Controls" } + elseif ($rc.path -match "src/Core/") { $area = "Core" } + elseif ($rc.path -match "src/Essentials/") { $area = "Essentials" } + elseif ($rc.path -match "src/BlazorWebView/") { $area = "BlazorWebView" } + elseif ($rc.path -match "\.github/") { $area = "GitHub/CI" } + elseif ($rc.path -match "eng/") { $area = "Engineering" } + else { $area = "Other" } + + if (-not $suggestionAnalysis.filePathPatterns.ContainsKey($area)) { + $suggestionAnalysis.filePathPatterns[$area] = 0 + } + $suggestionAnalysis.filePathPatterns[$area]++ + } + } + } + + # Calculate rates + $totalSuggestions = $suggestionAnalysis.totalCopilotSuggestions + if ($totalSuggestions -gt 0) { + $totalAccepted = ($data.sources.recentPRSuggestions | ForEach-Object { $_.suggestionStats.suggestionsAccepted } | Measure-Object -Sum).Sum + $totalRejected = ($data.sources.recentPRSuggestions | ForEach-Object { $_.suggestionStats.suggestionsRejected } | Measure-Object -Sum).Sum + $suggestionAnalysis.acceptanceRate = [Math]::Round(($totalAccepted / $totalSuggestions) * 100, 1) + $suggestionAnalysis.rejectionRate = [Math]::Round(($totalRejected / $totalSuggestions) * 100, 1) + } + + Write-Host " Recent PRs analyzed: $($suggestionAnalysis.totalPRs)" + Write-Host " Copilot PRs: $($suggestionAnalysis.copilotPRs)" + Write-Host " Review comments: $($suggestionAnalysis.totalReviewComments)" +} + +# ───────────────────────────────────────────────────────────── +# Generate recommendations +# ───────────────────────────────────────────────────────────── +Write-Host "`n💡 Generating recommendations..." -ForegroundColor Yellow + +$recommendations = @() + +# Recommendation: High failure rate patterns +if ($patterns.failedAttempts.Count -gt 0) { + $failedApproaches = $patterns.failedAttempts | Group-Object { $_.approach.Substring(0, [Math]::Min(50, $_.approach.Length)) } + foreach ($group in ($failedApproaches | Sort-Object Count -Descending | Select-Object -First 3)) { + $recommendations += @{ + category = "Instruction File" + priority = "Medium" + location = ".github/instructions/" + change = "Document that '$($group.Name)' approach has failed in $($group.Count) PR(s): $($group.Group.pr -join ', ')" + evidence = "PRs: $($group.Group.pr -join ', ')" + impact = "Prevents agents from repeating known-failing approaches" + } + } +} + +# Recommendation: Quick success patterns to reinforce +if ($patterns.singleAttempt.Count -gt 0) { + $recommendations += @{ + category = "Skill Enhancement" + priority = "Medium" + location = ".github/skills/try-fix/SKILL.md" + change = "Document quick-success patterns from PRs: $($patterns.singleAttempt.pr -join ', '). These succeeded on first attempt." + evidence = "PRs: $($patterns.singleAttempt.pr -join ', ')" + impact = "Reinforces successful search/fix strategies" + } +} + +# Recommendation: Multiple attempts indicate discovery issues +if ($patterns.multipleAttempts.Count -gt 0) { + $avgAttempts = ($patterns.multipleAttempts | Measure-Object -Property attempts -Average).Average + $recommendations += @{ + category = "Instruction File" + priority = "High" + location = ".github/copilot-instructions.md" + change = "PRs with slow discovery (avg $([Math]::Round($avgAttempts, 1)) attempts): $($patterns.multipleAttempts.pr -join ', '). Consider adding guidance for these component areas." + evidence = "PRs: $($patterns.multipleAttempts.pr -join ', ')" + impact = "Reduces average fix attempts from $([Math]::Round($avgAttempts, 1)) toward 1-2" + } +} + +# Recommendation: Root cause patterns +if ($patterns.rootCauses.Count -gt 0) { + $recommendations += @{ + category = "Architecture Doc" + priority = "Low" + location = ".github/instructions/" + change = "Document common root causes found across $($patterns.rootCauses.Count) PR(s) to help agents identify root causes faster." + evidence = "PRs: $($patterns.rootCauses.pr -join ', ')" + impact = "Helps agents identify root causes faster by learning from past findings" + } +} + +# Recommendation: If PRs lack test verification +if ($prsWithTryFix -gt 0 -and $prsWithTestVerification -eq 0) { + $recommendations += @{ + category = "Skill Enhancement" + priority = "High" + location = ".github/skills/try-fix/SKILL.md" + change = "Ensure try-fix attempts include test verification step. $prsWithTryFix PR(s) had try-fix but no test verification." + evidence = "Analysis of Copilot comments" + impact = "Increases confidence in fix correctness before reporting" + } +} + +# Memory-based recommendations +if ($memoryAnalysis.totalMemories -gt 0) { + # Check for frequently recurring memory subjects + $topSubjects = $memoryAnalysis.bySubject.GetEnumerator() | Sort-Object Value -Descending | Select-Object -First 3 + foreach ($subject in $topSubjects) { + if ($subject.Value -ge $MEMORY_FREQUENCY_THRESHOLD) { + $recommendations += @{ + category = "Instruction File" + priority = "Medium" + location = ".github/instructions/" + change = "Memory pattern '$($subject.Key)' appears $($subject.Value) times across sessions. Consider formalizing this as an instruction to prevent agents from having to re-learn it." + evidence = "Memory analysis: $($subject.Value) occurrences of '$($subject.Key)'" + impact = "Reduces re-learning overhead for agents by codifying frequently stored knowledge" + } + } + } + + # If many conventions are being stored, they should be in instruction files + if ($memoryAnalysis.conventions.Count -gt 0) { + $recommendations += @{ + category = "Instruction File" + priority = "High" + location = ".github/copilot-instructions.md" + change = "Found $($memoryAnalysis.conventions.Count) convention-related memory entries. These should be documented in instruction files so agents don't need to rely on memory recall." + evidence = "Memory scan: $($memoryAnalysis.conventions.Count) convention patterns found" + impact = "Ensures conventions are always available, not dependent on memory retention" + } + } +} + +# Recent PR suggestion-based recommendations +if ($suggestionAnalysis.totalPRs -gt 0) { + # High rejection rate indicates suggestion quality issues + if ($suggestionAnalysis.rejectionRate -gt $HIGH_REJECTION_THRESHOLD) { + $recommendations += @{ + category = "Instruction File" + priority = "High" + location = ".github/copilot-instructions.md" + change = "Copilot suggestions have a $($suggestionAnalysis.rejectionRate)% rejection rate across $($suggestionAnalysis.totalPRs) recent PRs. Review rejected suggestions to identify common issues and add guidance." + evidence = "Review comment analysis: $($suggestionAnalysis.totalCopilotSuggestions) suggestions, $($suggestionAnalysis.rejectionRate)% rejected" + impact = "Improves suggestion acceptance rate by addressing common rejection reasons" + } + } + + # High review comment volume in specific areas suggests missing guidance + $hotAreas = $suggestionAnalysis.filePathPatterns.GetEnumerator() | Sort-Object Value -Descending | Select-Object -First 3 + foreach ($area in $hotAreas) { + if ($area.Value -ge $HOTSPOT_COMMENT_THRESHOLD) { + $recommendations += @{ + category = "Instruction File" + priority = "Medium" + location = ".github/instructions/" + change = "Area '$($area.Key)' has $($area.Value) review comments across recent PRs. Consider adding or enhancing instruction files for this area." + evidence = "Review comment analysis: $($area.Value) comments in $($area.Key) files" + impact = "Reduces review friction in high-comment areas" + } + } + } + + # Copilot PRs success tracking + if ($suggestionAnalysis.copilotPRs -gt 0) { + $recommendations += @{ + category = "General" + priority = "Low" + location = ".github/copilot-instructions.md" + change = "Found $($suggestionAnalysis.copilotPRs) Copilot-labeled PR(s) in the last $($suggestionAnalysis.totalPRs) PRs. Track these for ongoing quality improvement." + evidence = "PRs: $(($suggestionAnalysis.copilotPRList | Select-Object -First 10 | ForEach-Object { "#$($_.number)" }) -join ', ')" + impact = "Provides baseline metrics for agent improvement tracking" + } + } +} + +# ───────────────────────────────────────────────────────────── +# Generate Report +# ───────────────────────────────────────────────────────────── +Write-Host "`n📝 Generating report..." -ForegroundColor Yellow + +$report = @" +# Scrape and Improve: Analysis Report + +**Generated:** $(Get-Date -Format "yyyy-MM-dd HH:mm:ss UTC") + +## Data Summary + +| Metric | Value | +|--------|-------| +| Total PRs Analyzed | $($data.summary.totalPRsAnalyzed) | +| Agent Session Files | $($data.summary.totalSessionFiles) | +| Copilot Comments | $($data.summary.totalCopilotComments) | +| CCA Sessions | $($data.summary.totalCCASessions) | +| Memories Collected | $($data.summary.totalMemories) | +| Recent PRs Scraped | $($data.summary.totalRecentPRsScraped) | +| Suggestion Responses | $($data.summary.totalSuggestionResponses) | +| Total Fix Attempts | $($data.summary.totalFixAttempts) | +| Successful Fixes | $($data.summary.totalSuccessfulFixes) | +| Failed Fixes | $($data.summary.totalFailedFixes) | +| Success Rate | $(if ($data.summary.totalFixAttempts -gt 0) { [Math]::Round(($data.summary.totalSuccessfulFixes / $data.summary.totalFixAttempts) * 100, 1) } else { "N/A" })% | + +## Pattern Analysis + +### PRs Requiring Multiple Attempts (Slow Discovery) + +$(if ($patterns.multipleAttempts.Count -gt 0) { + "| PR | Attempts | Session File |`n|-----|----------|--------------|" + $patterns.multipleAttempts | ForEach-Object { "| #$($_.pr) | $($_.attempts) | $($_.file) |" } +} else { + "_No PRs with multiple attempts found._" +}) + +### Quick Successes (First Attempt) + +$(if ($patterns.singleAttempt.Count -gt 0) { + "| PR | Session File |`n|-----|--------------|" + $patterns.singleAttempt | ForEach-Object { "| #$($_.pr) | $($_.file) |" } +} else { + "_No single-attempt successes found._" +}) + +### Copilot Integration Status + +| Metric | Count | +|--------|-------| +| PRs with AI Summary | $prsWithAISummary | +| PRs with Try-Fix | $prsWithTryFix | +| PRs with Test Verification | $prsWithTestVerification | + +### Root Causes Identified + +$(if ($patterns.rootCauses.Count -gt 0) { + $patterns.rootCauses | ForEach-Object { + "- **PR #$($_.pr)**: $($_.rootCause.Substring(0, [Math]::Min(200, $_.rootCause.Length)))..." + } +} else { + "_No root causes extracted._" +}) + +## Memory Analysis + +| Metric | Value | +|--------|-------| +| Total Memories | $($memoryAnalysis.totalMemories) | +| Unique Subjects | $($memoryAnalysis.bySubject.Count) | +| Convention Patterns | $($memoryAnalysis.conventions.Count) | +| Build Command Patterns | $($memoryAnalysis.buildCommands.Count) | + +$(if ($memoryAnalysis.bySubject.Count -gt 0) { + "### Memory Subject Frequency`n`n| Subject | Count |`n|---------|-------|" + $memoryAnalysis.bySubject.GetEnumerator() | Sort-Object Value -Descending | ForEach-Object { "| $($_.Key) | $($_.Value) |" } +} else { + "_No memory data available. Pass -MemoryContext parameter with stored memories to analyze._" +}) + +## Recent PR Suggestion Analysis + +| Metric | Value | +|--------|-------| +| Recent PRs Scraped | $($suggestionAnalysis.totalPRs) | +| Copilot PRs Found | $($suggestionAnalysis.copilotPRs) | +| Total Review Comments | $($suggestionAnalysis.totalReviewComments) | +| Copilot Suggestions | $($suggestionAnalysis.totalCopilotSuggestions) | +| Suggestion Acceptance Rate | $($suggestionAnalysis.acceptanceRate)% | +| Suggestion Rejection Rate | $($suggestionAnalysis.rejectionRate)% | + +$(if ($suggestionAnalysis.copilotPRList.Count -gt 0) { + "### Copilot PRs`n`n| PR | Title | State |`n|----|-------|-------|" + $suggestionAnalysis.copilotPRList | ForEach-Object { "| #$($_.number) | $($_.title) | $($_.state) |" } +}) + +$(if ($suggestionAnalysis.filePathPatterns.Count -gt 0) { + "### Review Comment Hotspots (by area)`n`n| Area | Review Comments |`n|------|----------------|" + $suggestionAnalysis.filePathPatterns.GetEnumerator() | Sort-Object Value -Descending | ForEach-Object { "| $($_.Key) | $($_.Value) |" } +}) + +## Recommendations + +$(if ($recommendations.Count -gt 0) { + $i = 0 + $recommendations | ForEach-Object { + $i++ + @" + +### $i. [$($_.priority)] $($_.category) + +- **Location:** ``$($_.location)`` +- **Change:** $($_.change) +- **Evidence:** $($_.evidence) +- **Impact:** $($_.impact) + +"@ + } +} else { + "_No recommendations generated. Consider collecting more data._" +}) + +## Existing Instruction Files + +$(if ($existingInstructions.Count -gt 0) { + $existingInstructions | ForEach-Object { "- ``$_``" } +} else { + "_No instruction files found._" +}) + +## Next Steps + +1. Review recommendations above and decide which to apply +2. For automated application, use the ``learn-from-pr`` agent +3. For manual application, edit the target files directly +4. Re-run this analysis periodically to track improvement +"@ + +$reportFile = Join-Path $OutputDir "analysis-report.md" +$report | Set-Content -Path $reportFile -Encoding UTF8 + +Write-Host "`n========================================" -ForegroundColor Green +Write-Host " Analysis Complete!" -ForegroundColor Green +Write-Host "========================================" -ForegroundColor Green +Write-Host " Recommendations: $($recommendations.Count)" +Write-Host " Report: $reportFile" -ForegroundColor Cyan + +# Also output recommendations as JSON for programmatic use +$recommendationsFile = Join-Path $OutputDir "recommendations.json" +$recommendations | ConvertTo-Json -Depth 5 | Set-Content -Path $recommendationsFile -Encoding UTF8 +Write-Host " Recommendations JSON: $recommendationsFile" -ForegroundColor Cyan diff --git a/.github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 b/.github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 new file mode 100644 index 000000000000..d97df92d07b4 --- /dev/null +++ b/.github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 @@ -0,0 +1,570 @@ +<# +.SYNOPSIS + Collects agent interaction data from multiple sources for analysis. + +.DESCRIPTION + Gathers data from: + - Agent PR session files (.github/agent-pr-session/*.md) + - Copilot comments on PRs (via gh CLI) + - CCA session logs (CustomAgentLogsTmp/PRState/) + - Repository memories (stored facts from agent interactions) + - Recent PRs with Copilot suggestion responses (review comments) + Outputs structured JSON for analysis by Analyze-And-Recommend.ps1. + +.PARAMETER PRNumbers + Comma-separated list of PR numbers to analyze. If not specified, discovers PRs automatically. + +.PARAMETER Label + GitHub label to filter PRs by (e.g., "copilot"). Default: "copilot" + +.PARAMETER Since + ISO 8601 date string to filter PRs from. Default: 30 days ago. + +.PARAMETER OutputDir + Output directory for collected data. Default: CustomAgentLogsTmp/scrape-and-improve + +.PARAMETER RepoRoot + Repository root directory. Default: current directory. + +.PARAMETER RecentPRCount + Number of most recent PRs to scrape for Copilot suggestion analysis. Default: 20. + +.PARAMETER MemoryContext + Raw memory context text to parse. If not provided, script looks for memory files in standard locations. + +.EXAMPLE + pwsh .github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 + pwsh .github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 -PRNumbers "33380,33134" + pwsh .github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 -Label "copilot" -Since "2026-01-01" + pwsh .github/skills/scrape-and-improve/scripts/Collect-AgentData.ps1 -RecentPRCount 20 +#> + +param( + [string]$PRNumbers = "", + [string]$Label = "copilot", + [string]$Since = "", + [string]$OutputDir = "CustomAgentLogsTmp/scrape-and-improve", + [string]$RepoRoot = ".", + [string]$Repository = "dotnet/maui", + [int]$RecentPRCount = 20, + [string]$MemoryContext = "" +) + +$ErrorActionPreference = "Continue" + +# Ensure output directory exists +New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host " Scrape and Improve: Data Collection" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan + +# Default Since to 30 days ago +if (-not $Since) { + $Since = (Get-Date).AddDays(-30).ToString("yyyy-MM-dd") +} + +$collectedData = @{ + collectedAt = (Get-Date -Format "yyyy-MM-ddTHH:mm:ssZ") + sources = @{ + agentSessions = @() + copilotComments = @() + ccaSessions = @() + memories = @() + recentPRSuggestions = @() + } + summary = @{ + totalPRsAnalyzed = 0 + totalSessionFiles = 0 + totalCopilotComments = 0 + totalCCASessions = 0 + totalFixAttempts = 0 + totalSuccessfulFixes = 0 + totalFailedFixes = 0 + totalMemories = 0 + totalRecentPRsScraped = 0 + totalSuggestionResponses = 0 + } +} + +# ───────────────────────────────────────────────────────────── +# Source 1: Agent PR Session files +# ───────────────────────────────────────────────────────────── +Write-Host "`n📁 Collecting Agent PR Session files..." -ForegroundColor Yellow + +$sessionDir = Join-Path $RepoRoot ".github/agent-pr-session" +if (Test-Path $sessionDir) { + $sessionFiles = Get-ChildItem -Path $sessionDir -Filter "*.md" -File + Write-Host " Found $($sessionFiles.Count) session file(s)" + + foreach ($file in $sessionFiles) { + $content = Get-Content -Path $file.FullName -Raw + $prNumber = "" + + # Extract PR number from filename (pr-XXXXX.md) + if ($file.Name -match "pr-(\d+)\.md") { + $prNumber = $Matches[1] + } + + # Extract key sections + $session = @{ + file = $file.Name + prNumber = $prNumber + phases = @() + fixCandidates = @() + recommendation = "" + rootCause = "" + } + + # Extract phase statuses + if ($content -match "(?s)\| Phase \| Status \|.*?\n((?:\|.*?\n)+)") { + $phaseLines = $Matches[1] -split "`n" | Where-Object { $_ -match "^\|" } + foreach ($line in $phaseLines) { + if ($line -match "\|\s*(.+?)\s*\|\s*(.+?)\s*\|") { + $session.phases += @{ + phase = $Matches[1].Trim() + status = $Matches[2].Trim() + } + } + } + } + + # Extract recommendation + if ($content -match "Recommendation:\s*(.*?)$" ) { + $session.recommendation = $Matches[1].Trim() + } + + # Extract fix candidates table + if ($content -match "(?s)\| # \| Source \| Approach \| Test Result \|.*?\n((?:\|.*?\n)+)") { + $fixLines = $Matches[1] -split "`n" | Where-Object { $_ -match "^\|" } + foreach ($line in $fixLines) { + $parts = ($line -split "\|" | Where-Object { $_.Trim() } | ForEach-Object { $_.Trim() }) + if ($parts.Count -ge 4) { + $result = $parts[3] + $isPass = $result -match "PASS" + $isFail = $result -match "FAIL" + $session.fixCandidates += @{ + number = $parts[0] + source = $parts[1] + approach = $parts[2] + result = $result + passed = $isPass + failed = $isFail + } + $collectedData.summary.totalFixAttempts++ + if ($isPass) { $collectedData.summary.totalSuccessfulFixes++ } + if ($isFail) { $collectedData.summary.totalFailedFixes++ } + } + } + } + + # Extract root cause + if ($content -match "(?s)Root Cause.*?\n(.*?)(?=\n#{2,}|\n\*\*|)") { + $session.rootCause = $Matches[1].Trim().Substring(0, [Math]::Min(500, $Matches[1].Trim().Length)) + } + + $collectedData.sources.agentSessions += $session + $collectedData.summary.totalSessionFiles++ + } +} else { + Write-Host " No agent-pr-session directory found" +} + +# ───────────────────────────────────────────────────────────── +# Source 2: Copilot Comments on PRs (via gh CLI) +# ───────────────────────────────────────────────────────────── +Write-Host "`n💬 Collecting Copilot comments from PRs..." -ForegroundColor Yellow + +# Check if gh CLI is available +$ghAvailable = $null -ne (Get-Command "gh" -ErrorAction SilentlyContinue) + +if ($ghAvailable) { + $prList = @() + + if ($PRNumbers) { + # Use specified PR numbers + $prList = $PRNumbers -split "," | ForEach-Object { $_.Trim() } + Write-Host " Using specified PRs: $($prList -join ', ')" + } else { + # Discover PRs with copilot label + Write-Host " Searching for PRs with label '$Label' since $Since..." + try { + $searchResult = gh pr list --repo $Repository --label $Label --state all --limit 50 --json number,title,state,createdAt 2>&1 + if ($LASTEXITCODE -eq 0 -and $searchResult) { + $prs = $searchResult | ConvertFrom-Json + $prList = $prs | Where-Object { + [datetime]$_.createdAt -ge [datetime]$Since + } | ForEach-Object { $_.number.ToString() } + Write-Host " Found $($prList.Count) PR(s) with label '$Label'" + } + } catch { + Write-Host " ⚠️ Could not search PRs: $_" -ForegroundColor Red + } + } + + foreach ($pr in $prList) { + Write-Host " Processing PR #$pr..." + try { + # Get PR comments + $comments = gh pr view $pr --repo $Repository --json comments --jq '.comments[] | {body: .body, author: .author.login, createdAt: .createdAt}' 2>&1 + if ($LASTEXITCODE -ne 0) { continue } + + $prCommentData = @{ + prNumber = $pr + aiSummaryFound = $false + tryFixAttempts = 0 + testVerification = $false + codeReview = $false + commentCount = 0 + patterns = @() + } + + # Parse comments as JSON lines + $commentObjects = @() + try { + $commentObjects = $comments | ConvertFrom-Json -ErrorAction SilentlyContinue + } catch { + # Try line-by-line parsing + } + + foreach ($comment in $commentObjects) { + $body = if ($comment.body) { $comment.body } else { "$comment" } + $prCommentData.commentCount++ + + # Check for AI Summary marker + if ($body -match "") { + $prCommentData.aiSummaryFound = $true + } + + # Count try-fix attempts + if ($body -match "Try-Fix Attempt") { + $prCommentData.tryFixAttempts++ + } + + # Check for test verification + if ($body -match "SECTION:VERIFY-TESTS|Test Verification") { + $prCommentData.testVerification = $true + } + + # Check for code review + if ($body -match "SECTION:PR-REVIEW|Code Review") { + $prCommentData.codeReview = $true + } + + # Look for failure patterns in comments + if ($body -match "(?i)wrong file|tunnel vision|misread error|over-engineer") { + $prCommentData.patterns += "failure-pattern-mentioned" + } + } + + $collectedData.sources.copilotComments += $prCommentData + $collectedData.summary.totalCopilotComments += $prCommentData.commentCount + + } catch { + Write-Host " ⚠️ Could not process PR #${pr}: $_" -ForegroundColor Red + } + } +} else { + Write-Host " ⚠️ gh CLI not available, skipping Copilot comment collection" -ForegroundColor Red +} + +# ───────────────────────────────────────────────────────────── +# Source 3: CCA Session Data +# ───────────────────────────────────────────────────────────── +Write-Host "`n📂 Collecting CCA session data..." -ForegroundColor Yellow + +$ccaDir = Join-Path $RepoRoot "CustomAgentLogsTmp/PRState" +if (Test-Path $ccaDir) { + $ccaFiles = Get-ChildItem -Path $ccaDir -Filter "*.md" -File -ErrorAction SilentlyContinue + Write-Host " Found $($ccaFiles.Count) CCA session file(s)" + + foreach ($file in $ccaFiles) { + $content = Get-Content -Path $file.FullName -Raw + $sessionData = @{ + file = $file.Name + issueOrPR = "" + hasFixCandidates = $false + attemptCount = 0 + phases = @() + } + + # Extract issue/PR number + if ($file.Name -match "(issue|pr)-(\d+)\.md") { + $sessionData.issueOrPR = "$($Matches[1])-$($Matches[2])" + } + + # Check for fix candidates + if ($content -match "Fix Candidates") { + $sessionData.hasFixCandidates = $true + } + + # Count attempts + $attemptMatches = [regex]::Matches($content, "(?i)attempt|try-fix") + $sessionData.attemptCount = $attemptMatches.Count + + # Extract phases + if ($content -match "(?s)\| Phase \| Status \|.*?\n((?:\|.*?\n)+)") { + $phaseLines = $Matches[1] -split "`n" | Where-Object { $_ -match "^\|" } + foreach ($line in $phaseLines) { + if ($line -match "\|\s*(.+?)\s*\|\s*(.+?)\s*\|") { + $sessionData.phases += @{ + phase = $Matches[1].Trim() + status = $Matches[2].Trim() + } + } + } + } + + $collectedData.sources.ccaSessions += $sessionData + $collectedData.summary.totalCCASessions++ + } + + # Also check for try-fix attempt directories + $tryFixDirs = Get-ChildItem -Path $ccaDir -Directory -ErrorAction SilentlyContinue | ForEach-Object { + $tryFixPath = Join-Path $_.FullName "try-fix" + if (Test-Path $tryFixPath) { + Get-ChildItem -Path $tryFixPath -Directory -Filter "attempt-*" -ErrorAction SilentlyContinue + } + } + + if ($tryFixDirs) { + Write-Host " Found $($tryFixDirs.Count) try-fix attempt directories" + } +} else { + Write-Host " No CCA session data directory found" +} + +# ───────────────────────────────────────────────────────────── +# Source 4: Repository Memories +# ───────────────────────────────────────────────────────────── +Write-Host "`n🧠 Collecting repository memories..." -ForegroundColor Yellow + +$memoryEntries = @() + +# Parse memory context if provided directly +if ($MemoryContext) { + Write-Host " Parsing provided memory context..." + # Parse structured memory blocks: **subject** followed by - Fact: and - Citations: + $memoryBlocks = [regex]::Matches($MemoryContext, '(?ms)\*\*(.+?)\*\*\s*\n- Fact:\s*(.+?)\n- Citations:\s*(.+?)(?=\n\n\*\*|\z)') + foreach ($block in $memoryBlocks) { + $memoryEntries += @{ + subject = $block.Groups[1].Value.Trim() + fact = $block.Groups[2].Value.Trim() + citations = $block.Groups[3].Value.Trim() + source = "provided-context" + } + } + Write-Host " Parsed $($memoryEntries.Count) memory entries from context" +} + +# Also scan for memory-like patterns in agent session files and CCA logs +$memoryPatterns = @( + @{ pattern = "(?i)store_memory|stored.*fact|memory.*stored"; type = "store-event" } + @{ pattern = "(?i)convention|best practice|always use|never use|must use"; type = "convention" } + @{ pattern = "(?i)build.*command|test.*command|run.*with"; type = "build-command" } +) + +# Check agent PR sessions for embedded learnings +$sessionDir = Join-Path $RepoRoot ".github/agent-pr-session" +if (Test-Path $sessionDir) { + $sessionFiles = Get-ChildItem -Path $sessionDir -Filter "*.md" -File + foreach ($file in $sessionFiles) { + $content = Get-Content -Path $file.FullName -Raw + foreach ($mp in $memoryPatterns) { + $matches = [regex]::Matches($content, $mp.pattern) + if ($matches.Count -gt 0) { + $memoryEntries += @{ + subject = $mp.type + fact = "Pattern '$($mp.type)' found $($matches.Count) time(s) in $($file.Name)" + citations = $file.Name + source = "agent-session-scan" + } + } + } + } +} + +# Check CCA logs for embedded learnings +$ccaDir = Join-Path $RepoRoot "CustomAgentLogsTmp/PRState" +if (Test-Path $ccaDir) { + $ccaFiles = Get-ChildItem -Path $ccaDir -Filter "*.md" -File -ErrorAction SilentlyContinue + foreach ($file in $ccaFiles) { + $content = Get-Content -Path $file.FullName -Raw + foreach ($mp in $memoryPatterns) { + $matches = [regex]::Matches($content, $mp.pattern) + if ($matches.Count -gt 0) { + $memoryEntries += @{ + subject = $mp.type + fact = "Pattern '$($mp.type)' found $($matches.Count) time(s) in $($file.Name)" + citations = "CCA: $($file.Name)" + source = "cca-session-scan" + } + } + } + } +} + +$collectedData.sources.memories = $memoryEntries +$collectedData.summary.totalMemories = $memoryEntries.Count +Write-Host " Total memory entries: $($memoryEntries.Count)" + +# ───────────────────────────────────────────────────────────── +# Source 5: Recent PRs - Copilot Suggestion Responses +# ───────────────────────────────────────────────────────────── +Write-Host "`n📊 Scraping recent PRs for Copilot suggestion analysis..." -ForegroundColor Yellow + +if ($ghAvailable) { + Write-Host " Fetching $RecentPRCount most recent PRs..." + try { + $recentPRsJson = gh pr list --repo $Repository --state all --limit $RecentPRCount --json number,title,author,state,createdAt,labels,reviewDecision 2>&1 + if ($LASTEXITCODE -eq 0 -and $recentPRsJson) { + $recentPRs = $recentPRsJson | ConvertFrom-Json + Write-Host " Found $($recentPRs.Count) recent PR(s)" + + foreach ($rpr in $recentPRs) { + $prNum = $rpr.number + Write-Host " Analyzing PR #$prNum review comments..." + + $prSuggestionData = @{ + prNumber = $prNum + title = $rpr.title + author = if ($rpr.author) { $rpr.author.login } else { "" } + state = $rpr.state + labels = @() + isCopilotPR = $false + reviewComments = @() + suggestionStats = @{ + totalReviewComments = 0 + copilotSuggestions = 0 + suggestionsAccepted = 0 + suggestionsRejected = 0 + suggestionsDiscussed = 0 + codeChangeRequests = 0 + } + } + + # Extract labels + if ($rpr.labels) { + $prSuggestionData.labels = @($rpr.labels | ForEach-Object { $_.name }) + $prSuggestionData.isCopilotPR = $prSuggestionData.labels -contains "copilot" + } + + # Get review comments (inline code review feedback) + try { + $reviewCommentsJson = gh api "repos/$Repository/pulls/$prNum/comments" --paginate 2>&1 + if ($LASTEXITCODE -eq 0 -and $reviewCommentsJson) { + $reviewComments = $reviewCommentsJson | ConvertFrom-Json -ErrorAction SilentlyContinue + + foreach ($rc in $reviewComments) { + $prSuggestionData.suggestionStats.totalReviewComments++ + + $commentInfo = @{ + author = if ($rc.user) { $rc.user.login } else { "" } + body = if ($rc.body.Length -gt 500) { $rc.body.Substring(0, 500) } else { $rc.body } + path = $rc.path + createdAt = $rc.created_at + isCopilot = $false + isAccepted = $false + isRejected = $false + hasSuggestion = $false + } + + # Detect Copilot-authored comments (specific bot accounts) + if ($commentInfo.author -match "^(copilot|github-copilot)\[bot\]$|^copilot$") { + $commentInfo.isCopilot = $true + $prSuggestionData.suggestionStats.copilotSuggestions++ + } + + # Detect suggestion blocks + if ($rc.body -match '```suggestion') { + $commentInfo.hasSuggestion = $true + } + + # Detect acceptance patterns (resolved, committed suggestion) + $isAcceptMatch = $rc.body -match '(?i)lgtm|looks good|approved|applied|makes sense|good catch|fixed' + # Detect rejection patterns + $isRejectMatch = $rc.body -match "(?i)disagree|won't fix|not needed|revert|nack|don't think|shouldn't|incorrect" + + # Avoid double-counting: prioritize rejection over acceptance + if ($isRejectMatch) { + $commentInfo.isRejected = $true + $prSuggestionData.suggestionStats.suggestionsRejected++ + } elseif ($isAcceptMatch) { + $commentInfo.isAccepted = $true + $prSuggestionData.suggestionStats.suggestionsAccepted++ + } + + # Detect change requests + if ($rc.body -match '(?i)please change|should be|needs to be|consider using|instead of') { + $prSuggestionData.suggestionStats.codeChangeRequests++ + } + + # Count discussions (replies that aren't simple acceptance/rejection) + if (-not $commentInfo.isAccepted -and -not $commentInfo.isRejected -and $rc.body.Length -gt 50) { + $prSuggestionData.suggestionStats.suggestionsDiscussed++ + } + + $prSuggestionData.reviewComments += $commentInfo + } + } + } catch { + Write-Host " ⚠️ Could not fetch review comments for PR #${prNum}: $_" -ForegroundColor Red + } + + # Get PR reviews (approve/request changes/comment) + try { + $reviewsJson = gh api "repos/$Repository/pulls/$prNum/reviews" 2>&1 + if ($LASTEXITCODE -eq 0 -and $reviewsJson) { + $reviews = $reviewsJson | ConvertFrom-Json -ErrorAction SilentlyContinue + foreach ($review in $reviews) { + if ($review.state -eq "CHANGES_REQUESTED") { + $prSuggestionData.suggestionStats.codeChangeRequests++ + } + } + } + } catch { + # Silently continue - reviews are supplementary + } + + $collectedData.sources.recentPRSuggestions += $prSuggestionData + $collectedData.summary.totalRecentPRsScraped++ + $collectedData.summary.totalSuggestionResponses += $prSuggestionData.suggestionStats.totalReviewComments + } + } + } catch { + Write-Host " ⚠️ Could not fetch recent PRs: $_" -ForegroundColor Red + } +} else { + Write-Host " ⚠️ gh CLI not available, skipping recent PR analysis" -ForegroundColor Red +} + +# ───────────────────────────────────────────────────────────── +# Compute totals +# ───────────────────────────────────────────────────────────── +$allPRs = @() +$allPRs += $collectedData.sources.agentSessions | ForEach-Object { $_.prNumber } | Where-Object { $_ } +$allPRs += $collectedData.sources.copilotComments | ForEach-Object { $_.prNumber } | Where-Object { $_ } +$allPRs += $collectedData.sources.recentPRSuggestions | ForEach-Object { $_.prNumber.ToString() } | Where-Object { $_ } +$collectedData.summary.totalPRsAnalyzed = ($allPRs | Sort-Object -Unique).Count + +# ───────────────────────────────────────────────────────────── +# Save output +# ───────────────────────────────────────────────────────────── +$outputFile = Join-Path $OutputDir "collected-data.json" +$collectedData | ConvertTo-Json -Depth 10 | Set-Content -Path $outputFile -Encoding UTF8 + +Write-Host "`n========================================" -ForegroundColor Green +Write-Host " Collection Complete!" -ForegroundColor Green +Write-Host "========================================" -ForegroundColor Green +Write-Host " Session files: $($collectedData.summary.totalSessionFiles)" +Write-Host " Copilot comments: $($collectedData.summary.totalCopilotComments)" +Write-Host " CCA sessions: $($collectedData.summary.totalCCASessions)" +Write-Host " Memories: $($collectedData.summary.totalMemories)" +Write-Host " Recent PRs scraped: $($collectedData.summary.totalRecentPRsScraped)" +Write-Host " Suggestion responses: $($collectedData.summary.totalSuggestionResponses)" +Write-Host " Total PRs analyzed: $($collectedData.summary.totalPRsAnalyzed)" +Write-Host " Fix attempts: $($collectedData.summary.totalFixAttempts)" +Write-Host " Successful: $($collectedData.summary.totalSuccessfulFixes)" +Write-Host " Failed: $($collectedData.summary.totalFailedFixes)" +Write-Host "" +Write-Host " Output: $outputFile" -ForegroundColor Cyan diff --git a/.github/workflows/scrape-and-improve.yml b/.github/workflows/scrape-and-improve.yml new file mode 100644 index 000000000000..84dd0cd8cc19 --- /dev/null +++ b/.github/workflows/scrape-and-improve.yml @@ -0,0 +1,66 @@ +name: "Scrape and Improve Agent Instructions" + +on: + # Run on a weekly schedule (every Monday at 9 AM UTC) + schedule: + - cron: '0 9 * * 1' + + # Allow manual trigger with parameters + workflow_dispatch: + inputs: + pr_numbers: + description: 'Comma-separated PR numbers to analyze (optional)' + required: false + type: string + label: + description: 'GitHub label to filter PRs by' + required: false + type: string + default: 'copilot' + since: + description: 'ISO 8601 date to filter PRs from (e.g., 2026-01-01)' + required: false + type: string + recent_pr_count: + description: 'Number of most recent PRs to scrape (default: 20)' + required: false + type: string + default: '20' + +permissions: + contents: read + issues: read + pull-requests: read + +jobs: + scrape-and-improve: + name: Analyze Agent Interactions + runs-on: ubuntu-latest + if: github.repository_owner == 'dotnet' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Run Scrape and Improve + id: scrape + uses: ./.github/actions/scrape-and-improve + with: + pr-numbers: ${{ inputs.pr_numbers || '' }} + label: ${{ inputs.label || 'copilot' }} + since: ${{ inputs.since || '' }} + recent-pr-count: ${{ inputs.recent_pr_count || '20' }} + + - name: Post summary to workflow + shell: pwsh + run: | + $reportFile = "${{ steps.scrape.outputs.report-file }}" + if (Test-Path $reportFile) { + $content = Get-Content -Path $reportFile -Raw + # Write to GitHub step summary + $content >> $env:GITHUB_STEP_SUMMARY + } + + Write-Host "✅ Analysis complete!" + Write-Host " Recommendations: ${{ steps.scrape.outputs.recommendations-count }}" + Write-Host " Report artifact: scrape-and-improve-results"