From 6d8c6cc147eb6e67ea77f4bc2120aa6ae41b341f Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 20 May 2026 19:50:33 -0500 Subject: [PATCH 1/4] feat(fleet-monitor): raise issues in the repo they pertain to - Track high-failure workflows: create/update issues in the target repo (owner/repo parsed from item.repo) instead of .github-private. Switch token from github.token to GH_PAT_WORKFLOWS for cross-repo issues:write. Ensure labels once per target repo via a Set guard. - Migrate misaligned issues: new step before tracking runs a one-shot migration using GraphQL transferIssue to move any open fleet-tracker issues currently in .github-private to their correct target repos. createLabelsIfMissing copies labels automatically. Soft-fails per issue so a single inaccessible repo can't block the whole step. - Issue lookup (fleet_monitor.sh step 2b): switch from querying only GITHUB_REPOSITORY to an org-wide search/issues query so the report's inline issue links resolve correctly after migration. Closes #315 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/actions-fleet-monitor.yml | 116 +++++++++++++++----- scripts/fleet_monitor.sh | 17 ++- 2 files changed, 99 insertions(+), 34 deletions(-) diff --git a/.github/workflows/actions-fleet-monitor.yml b/.github/workflows/actions-fleet-monitor.yml index b63287e03..c53744564 100644 --- a/.github/workflows/actions-fleet-monitor.yml +++ b/.github/workflows/actions-fleet-monitor.yml @@ -6,10 +6,11 @@ # • This is the canonical fleet-monitor implementation; there is no external # reusable. The scanning logic lives in scripts/fleet_monitor.sh and # scripts/fleet_report.sh. -# • The "Track high-failure workflows as Issues" step uses github.token (not -# GH_PAT_WORKFLOWS) because github.token has explicit issues:write on this -# repo. Dev-Lead Agent is explicitly woken via repository_dispatch (which -# requires contents:write) using GH_PAT_WORKFLOWS in the step below. +# • The "Migrate misaligned fleet-tracker issues" and "Track high-failure +# workflows as Issues" steps both use GH_PAT_WORKFLOWS (not github.token) +# because they need cross-repo issues:write on target repos. +# github.token only has issues:write on .github-private itself. +# Dev-Lead Agent is woken via repository_dispatch using GH_PAT_WORKFLOWS. # • Do NOT change checkout ref from 'main' — the github.ref_name variant is # for feature-branch testing only and must never reach production. # ───────────────────────────────────────────────────────────────────────────── @@ -99,14 +100,70 @@ jobs: core.notice(`Issue created: ${issue.data.html_url}`); + - name: Migrate misaligned fleet-tracker issues + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ secrets.GH_PAT_WORKFLOWS }} + script: | + // Find open fleet-tracker issues in .github-private that belong to other repos + const allIssues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + labels: 'fleet-tracker', + state: 'open', + per_page: 100, + }); + + let migrated = 0; + for (const issue of allIssues) { + if (!issue.title.startsWith('[Fleet Monitor] ')) continue; + + // Parse: "[Fleet Monitor] owner/repo — workflow.yml" + const match = issue.title.match(/^\[Fleet Monitor\] ([^/]+\/\S+) — (.+)$/); + if (!match) continue; + + const [, targetFullRepo] = match; + const [targetOwner, targetRepoName] = targetFullRepo.split('/'); + + // Already in the right repo — skip + if (targetRepoName === context.repo.repo && targetOwner === context.repo.owner) continue; + + core.info(`Migrating #${issue.number} "${issue.title}" → ${targetOwner}/${targetRepoName}`); + try { + const { repository } = await github.graphql(` + query($owner: String!, $name: String!) { + repository(owner: $owner, name: $name) { id } + } + `, { owner: targetOwner, name: targetRepoName }); + + const result = await github.graphql(` + mutation($issueId: ID!, $repoId: ID!) { + transferIssue(input: { + issueId: $issueId, + repositoryId: $repoId, + createLabelsIfMissing: true + }) { + issue { url number } + } + } + `, { issueId: issue.node_id, repoId: repository.id }); + + core.notice(`Transferred #${issue.number} → ${result.transferIssue.issue.url}`); + migrated++; + } catch (err) { + core.warning(`Failed to migrate #${issue.number}: ${err.message}`); + } + } + core.info(`Migration complete: ${migrated} issue(s) transferred.`); + - name: Track high-failure workflows as Issues if: hashFiles('fleet_high_failure.json') != '' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: ORG: ${{ env.ORG }} with: - # Use GITHUB_TOKEN (has explicit issues:write) for label creation and assignment - github-token: ${{ github.token }} + # Use GH_PAT_WORKFLOWS for cross-repo issues:write on target repos + github-token: ${{ secrets.GH_PAT_WORKFLOWS }} script: | const fs = require('fs'); @@ -117,24 +174,34 @@ jobs: return; } - // Ensure required labels exist (422 = already exists — safe to ignore) - for (const [name, color, desc] of [ - ['dev-lead', 'd93f0b', 'Requires dev-lead attention'], - ['fleet-tracker', '0075ca', 'Tracked by the Actions Fleet Monitor'], - ]) { - try { - await github.rest.issues.createLabel({ - owner: context.repo.owner, repo: context.repo.repo, - name, color, description: desc, - }); - } catch (e) { if (e.status !== 422) throw e; } - } - const labelEmoji = { CRITICAL: '🔴', DEGRADED: '🟠', WARNING: '🟡', ERROR: '⚫' }; const today = new Date().toISOString().split('T')[0]; const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; + // Track repos where we have already ensured labels exist + const labelledRepos = new Set(); + for (const item of items) { + // Issues are created in the repo they pertain to, not in .github-private + const [targetOwner, targetRepo] = item.repo.split('/'); + + // Ensure required labels exist in the target repo (once per repo) + if (!labelledRepos.has(item.repo)) { + for (const [name, color, desc] of [ + ['dev-lead', 'd93f0b', 'Requires dev-lead attention'], + ['fleet-tracker', '0075ca', 'Tracked by the Actions Fleet Monitor'], + ['health-check', '0e8a16', 'Automated health check result'], + ]) { + try { + await github.rest.issues.createLabel({ + owner: targetOwner, repo: targetRepo, + name, color, description: desc, + }); + } catch (e) { if (e.status !== 422) throw e; } + } + labelledRepos.add(item.repo); + } + const emoji = labelEmoji[item.label] ?? '🔴'; const title = `[Fleet Monitor] ${item.repo} — ${item.workflow}`; const body = [ @@ -155,10 +222,10 @@ jobs: `> _Threshold: failure rate > 10% over the monitored window._`, ].join('\n'); - // Search for an existing open issue with this exact title + // Search for an existing open issue in the target repo const { data: results } = await github.rest.search.issuesAndPullRequests({ q: [ - `repo:${context.repo.owner}/${context.repo.repo}`, + `repo:${targetOwner}/${targetRepo}`, 'is:issue', 'is:open', `in:title`, @@ -169,20 +236,19 @@ jobs: if (existing) { await github.rest.issues.update({ - owner: context.repo.owner, repo: context.repo.repo, + owner: targetOwner, repo: targetRepo, issue_number: existing.number, body, }); - // Ensure dev-lead label is present (addLabels is idempotent) await github.rest.issues.addLabels({ - owner: context.repo.owner, repo: context.repo.repo, + owner: targetOwner, repo: targetRepo, issue_number: existing.number, labels: ['dev-lead', 'fleet-tracker', 'health-check'], }); core.notice(`Updated issue #${existing.number}: ${existing.html_url}`); } else { const created = await github.rest.issues.create({ - owner: context.repo.owner, repo: context.repo.repo, + owner: targetOwner, repo: targetRepo, title, body, labels: ['dev-lead', 'fleet-tracker', 'health-check'], diff --git a/scripts/fleet_monitor.sh b/scripts/fleet_monitor.sh index 808642c80..50719ab9b 100755 --- a/scripts/fleet_monitor.sh +++ b/scripts/fleet_monitor.sh @@ -177,17 +177,16 @@ done # --------------------------------------------------------------------------- # 2b. Build issue lookup for open fleet-tracker issues (linked inline in report) +# Issues now live in their target repo, so search org-wide. # --------------------------------------------------------------------------- issues_lookup_file=$(mktemp) -if [ -n "${GITHUB_REPOSITORY:-}" ]; then - gh api "repos/${GITHUB_REPOSITORY}/issues?labels=fleet-tracker&state=open&per_page=100" \ - --paginate \ - --jq '.[] | select(.title | startswith("[Fleet Monitor] ")) | - (.title | ltrimstr("[Fleet Monitor] ") | split(" — ")) as $p | - select(($p | length) == 2) | - [$p[0], $p[1], .html_url, (.number | tostring)] | @tsv' \ - > "$issues_lookup_file" 2>/dev/null || true -fi +gh api "search/issues?q=org:${ORG}+label:fleet-tracker+is:open&per_page=100" \ + --paginate \ + --jq '.items[] | select(.title | startswith("[Fleet Monitor] ")) | + (.title | ltrimstr("[Fleet Monitor] ") | split(" — ")) as $p | + select(($p | length) == 2) | + [$p[0], $p[1], .html_url, (.number | tostring)] | @tsv' \ + > "$issues_lookup_file" 2>/dev/null || true # --------------------------------------------------------------------------- # 3. Generate reports From 1b060a81c8be58aa5169bc0c66c1f1b87a8e3f79 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 20 May 2026 19:54:33 -0500 Subject: [PATCH 2/4] fix(fleet-monitor): address Copilot review comments - fleet_monitor.sh: add is:issue to org-wide search query to exclude PRs with the fleet-tracker label from the issue lookup table - actions-fleet-monitor.yml: wrap per-item tracking loop in try/catch so one inaccessible or missing target repo can't block issue tracking for all other high-failure workflows Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/actions-fleet-monitor.yml | 135 ++++++++++---------- scripts/fleet_monitor.sh | 2 +- 2 files changed, 70 insertions(+), 67 deletions(-) diff --git a/.github/workflows/actions-fleet-monitor.yml b/.github/workflows/actions-fleet-monitor.yml index c53744564..e1310cb2c 100644 --- a/.github/workflows/actions-fleet-monitor.yml +++ b/.github/workflows/actions-fleet-monitor.yml @@ -184,76 +184,79 @@ jobs: for (const item of items) { // Issues are created in the repo they pertain to, not in .github-private const [targetOwner, targetRepo] = item.repo.split('/'); - - // Ensure required labels exist in the target repo (once per repo) - if (!labelledRepos.has(item.repo)) { - for (const [name, color, desc] of [ - ['dev-lead', 'd93f0b', 'Requires dev-lead attention'], - ['fleet-tracker', '0075ca', 'Tracked by the Actions Fleet Monitor'], - ['health-check', '0e8a16', 'Automated health check result'], - ]) { - try { - await github.rest.issues.createLabel({ - owner: targetOwner, repo: targetRepo, - name, color, description: desc, - }); - } catch (e) { if (e.status !== 422) throw e; } + try { + // Ensure required labels exist in the target repo (once per repo) + if (!labelledRepos.has(item.repo)) { + for (const [name, color, desc] of [ + ['dev-lead', 'd93f0b', 'Requires dev-lead attention'], + ['fleet-tracker', '0075ca', 'Tracked by the Actions Fleet Monitor'], + ['health-check', '0e8a16', 'Automated health check result'], + ]) { + try { + await github.rest.issues.createLabel({ + owner: targetOwner, repo: targetRepo, + name, color, description: desc, + }); + } catch (e) { if (e.status !== 422) throw e; } + } + labelledRepos.add(item.repo); } - labelledRepos.add(item.repo); - } - - const emoji = labelEmoji[item.label] ?? '🔴'; - const title = `[Fleet Monitor] ${item.repo} — ${item.workflow}`; - const body = [ - `## ${emoji} ${item.label}: \`${item.repo}\` / \`${item.workflow}\``, - '', - `| Metric | Value |`, - `|---|---|`, - `| **Failure Rate** | ${item.rate} (${item.failed} / ${item.total} runs) |`, - `| **Status** | ${item.label} |`, - `| **p50 Duration** | ${item.p50}s |`, - `| **p95 Duration** | ${item.p95}s |`, - `| **Successful Runs** | ${item.success} |`, - `| **Cancelled Runs** | ${item.cancelled} |`, - '', - `**Workflow:** [\`${item.workflow}\`](https://github.com/${item.repo}/actions)`, - '', - `> _Last updated by [Fleet Monitor run](${runUrl}) on ${today}._`, - `> _Threshold: failure rate > 10% over the monitored window._`, - ].join('\n'); - // Search for an existing open issue in the target repo - const { data: results } = await github.rest.search.issuesAndPullRequests({ - q: [ - `repo:${targetOwner}/${targetRepo}`, - 'is:issue', - 'is:open', - `in:title`, - `"${title}"`, - ].join(' '), - }); - const existing = results.items.find(i => i.title === title); + const emoji = labelEmoji[item.label] ?? '🔴'; + const title = `[Fleet Monitor] ${item.repo} — ${item.workflow}`; + const body = [ + `## ${emoji} ${item.label}: \`${item.repo}\` / \`${item.workflow}\``, + '', + `| Metric | Value |`, + `|---|---|`, + `| **Failure Rate** | ${item.rate} (${item.failed} / ${item.total} runs) |`, + `| **Status** | ${item.label} |`, + `| **p50 Duration** | ${item.p50}s |`, + `| **p95 Duration** | ${item.p95}s |`, + `| **Successful Runs** | ${item.success} |`, + `| **Cancelled Runs** | ${item.cancelled} |`, + '', + `**Workflow:** [\`${item.workflow}\`](https://github.com/${item.repo}/actions)`, + '', + `> _Last updated by [Fleet Monitor run](${runUrl}) on ${today}._`, + `> _Threshold: failure rate > 10% over the monitored window._`, + ].join('\n'); - if (existing) { - await github.rest.issues.update({ - owner: targetOwner, repo: targetRepo, - issue_number: existing.number, - body, + // Search for an existing open issue in the target repo + const { data: results } = await github.rest.search.issuesAndPullRequests({ + q: [ + `repo:${targetOwner}/${targetRepo}`, + 'is:issue', + 'is:open', + `in:title`, + `"${title}"`, + ].join(' '), }); - await github.rest.issues.addLabels({ - owner: targetOwner, repo: targetRepo, - issue_number: existing.number, - labels: ['dev-lead', 'fleet-tracker', 'health-check'], - }); - core.notice(`Updated issue #${existing.number}: ${existing.html_url}`); - } else { - const created = await github.rest.issues.create({ - owner: targetOwner, repo: targetRepo, - title, - body, - labels: ['dev-lead', 'fleet-tracker', 'health-check'], - }); - core.notice(`Created issue #${created.data.number}: ${created.data.html_url}`); + const existing = results.items.find(i => i.title === title); + + if (existing) { + await github.rest.issues.update({ + owner: targetOwner, repo: targetRepo, + issue_number: existing.number, + body, + }); + await github.rest.issues.addLabels({ + owner: targetOwner, repo: targetRepo, + issue_number: existing.number, + labels: ['dev-lead', 'fleet-tracker', 'health-check'], + }); + core.notice(`Updated issue #${existing.number}: ${existing.html_url}`); + } else { + const created = await github.rest.issues.create({ + owner: targetOwner, repo: targetRepo, + title, + body, + labels: ['dev-lead', 'fleet-tracker', 'health-check'], + }); + core.notice(`Created issue #${created.data.number}: ${created.data.html_url}`); + } + } catch (err) { + core.warning(`Failed to track ${item.repo}/${item.workflow}: ${err.message}`); } } diff --git a/scripts/fleet_monitor.sh b/scripts/fleet_monitor.sh index 50719ab9b..813319d01 100755 --- a/scripts/fleet_monitor.sh +++ b/scripts/fleet_monitor.sh @@ -180,7 +180,7 @@ done # Issues now live in their target repo, so search org-wide. # --------------------------------------------------------------------------- issues_lookup_file=$(mktemp) -gh api "search/issues?q=org:${ORG}+label:fleet-tracker+is:open&per_page=100" \ +gh api "search/issues?q=org:${ORG}+label:fleet-tracker+is:open+is:issue&per_page=100" \ --paginate \ --jq '.items[] | select(.title | startswith("[Fleet Monitor] ")) | (.title | ltrimstr("[Fleet Monitor] ") | split(" — ")) as $p | From 754caeb1be7349f4e97e7c10a2db8ba65f1cb3a0 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 20 May 2026 20:02:41 -0500 Subject: [PATCH 3/4] fix(ci): resolve yamllint errors in issue-triage-runner.yml - Replace '--only-binary :all:' with '--only-binary=":all:"' to avoid yamllint misinterpreting the bare ':all:' as a YAML mapping value - Remove extra alignment spaces in env variable keys (colons rule) - Replace heredoc Python snippet with python3 -c one-liner to prevent non-indented heredoc content from breaking YAML block scalar parsing Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/issue-triage-runner.yml | 29 ++++++++++------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/.github/workflows/issue-triage-runner.yml b/.github/workflows/issue-triage-runner.yml index e2ca17771..527eed603 100644 --- a/.github/workflows/issue-triage-runner.yml +++ b/.github/workflows/issue-triage-runner.yml @@ -27,27 +27,22 @@ jobs: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - name: Install Python dependencies - run: pip install "pyyaml==6.0.3" --only-binary :all: --quiet + run: pip install "pyyaml==6.0.3" --only-binary=":all:" --quiet - name: Run issue-triage (staged) id: run env: - ISSUE_NUMBER: ${{ github.event.issue.number }} - ISSUE_TITLE: ${{ github.event.issue.title }} - ISSUE_BODY: ${{ github.event.issue.body }} - ISSUE_LABELS: ${{ join(github.event.issue.labels.*.name, ',') }} - ISSUE_URL: ${{ github.event.issue.html_url }} - ISSUE_ACTION: ${{ github.event.action }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + ISSUE_TITLE: ${{ github.event.issue.title }} + ISSUE_BODY: ${{ github.event.issue.body }} + ISSUE_LABELS: ${{ join(github.event.issue.labels.*.name, ',') }} + ISSUE_URL: ${{ github.event.issue.html_url }} + ISSUE_ACTION: ${{ github.event.action }} GITHUB_REPOSITORY: ${{ github.repository }} run: | set -euo pipefail result=$(bash scripts/aw.sh run issue-triage --staged) - skip=$(AW_RESULT="$result" python3 - <<'PYEOF' -import json, os -d = json.loads(os.environ['AW_RESULT']) -print('true' if d.get('skip') else 'false') -PYEOF -) + skip=$(AW_RESULT="$result" python3 -c "import json,os; d=json.loads(os.environ['AW_RESULT']); print('true' if d.get('skip') else 'false')") # Store encoded to survive multi-line JSON in GITHUB_OUTPUT encoded=$(printf '%s\n' "$result" | base64 -w0) echo "result=$encoded" >> "$GITHUB_OUTPUT" @@ -67,14 +62,14 @@ PYEOF persist-credentials: false - name: Install Python dependencies - run: pip install "pyyaml==6.0.3" --only-binary :all: --quiet + run: pip install "pyyaml==6.0.3" --only-binary=":all:" --quiet - name: Apply triage output (safe-output) env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} - ISSUE_NUMBER: ${{ github.event.issue.number }} - TRIAGE_RESULT: ${{ needs.triage.outputs.result }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + TRIAGE_RESULT: ${{ needs.triage.outputs.result }} run: | result_file=$(mktemp) printf '%s\n' "$TRIAGE_RESULT" | base64 -d > "$result_file" From 8c7bf0b21421c21b2db4592108930bb1bc8be402 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 20 May 2026 20:06:02 -0500 Subject: [PATCH 4/4] fix(fleet-monitor): address CodeRabbit and ChatGPT Codex review comments - fleet_monitor.sh: replace silent '2>/dev/null || true' on search/issues with explicit exit-status check; emit ::warning:: annotation when the lookup fails so operators know inline links may be incomplete; add comment documenting the Search API 1,000-result cap - actions-fleet-monitor.yml: expand createLabel error handling from 'throw on non-422' to explicitly handle 403 (no write access) and 410 (issues disabled) with targeted core.warning messages instead of propagating to the outer catch; unexpected errors still re-throw Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/actions-fleet-monitor.yml | 10 +++++++++- scripts/fleet_monitor.sh | 11 +++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/actions-fleet-monitor.yml b/.github/workflows/actions-fleet-monitor.yml index e1310cb2c..8b4777b55 100644 --- a/.github/workflows/actions-fleet-monitor.yml +++ b/.github/workflows/actions-fleet-monitor.yml @@ -197,7 +197,15 @@ jobs: owner: targetOwner, repo: targetRepo, name, color, description: desc, }); - } catch (e) { if (e.status !== 422) throw e; } + } catch (e) { + if (e.status === 422) { /* already exists — ok */ } + else if (e.status === 403 || e.status === 410) { + // 403: no write access; 410: issues disabled on this repo + core.warning(`Cannot create label '${name}' in ${targetOwner}/${targetRepo} (${e.status}) — skipping`); + } else { + throw e; // propagate to outer catch for unexpected errors + } + } } labelledRepos.add(item.repo); } diff --git a/scripts/fleet_monitor.sh b/scripts/fleet_monitor.sh index 813319d01..3212ac11d 100755 --- a/scripts/fleet_monitor.sh +++ b/scripts/fleet_monitor.sh @@ -178,15 +178,22 @@ done # --------------------------------------------------------------------------- # 2b. Build issue lookup for open fleet-tracker issues (linked inline in report) # Issues now live in their target repo, so search org-wide. +# Note: the Search API caps at 1,000 results total; if the org has >1,000 open +# fleet-tracker issues the lookup may be incomplete, but that is not expected in +# practice. On search failure (auth error, rate-limit, etc.) inline links will +# simply be absent from the report — non-fatal for the monitoring workflow. # --------------------------------------------------------------------------- issues_lookup_file=$(mktemp) -gh api "search/issues?q=org:${ORG}+label:fleet-tracker+is:open+is:issue&per_page=100" \ +if ! gh api "search/issues?q=org:${ORG}+label:fleet-tracker+is:open+is:issue&per_page=100" \ --paginate \ --jq '.items[] | select(.title | startswith("[Fleet Monitor] ")) | (.title | ltrimstr("[Fleet Monitor] ") | split(" — ")) as $p | select(($p | length) == 2) | [$p[0], $p[1], .html_url, (.number | tostring)] | @tsv' \ - > "$issues_lookup_file" 2>/dev/null || true + > "$issues_lookup_file" 2>&1; then + echo "::warning::Could not fetch fleet-tracker issues org-wide — inline issue links may be incomplete. Check token auth/rate-limit." + : > "$issues_lookup_file" # ensure file exists but is empty on failure +fi # --------------------------------------------------------------------------- # 3. Generate reports