diff --git a/.github/workflows/actions-fleet-monitor.yml b/.github/workflows/actions-fleet-monitor.yml index aaf16d5d6..bf77e5b32 100644 --- a/.github/workflows/actions-fleet-monitor.yml +++ b/.github/workflows/actions-fleet-monitor.yml @@ -174,10 +174,31 @@ jobs: `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})._ `; } const today = new Date().toISOString().split('T')[0]; + const titlePrefix = `${process.env.ORG} — workflow failures detected`; + + // This daily digest embeds the date in its title, so a fresh issue is + // created every run — 42 had accumulated open. Close the prior digests + // before opening today's so only the current snapshot stays open (the + // digest is a point-in-time report; history lives in closed issues). + const priors = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, repo: context.repo.repo, + state: 'open', labels: 'health-check', per_page: 100, + }); + for (const p of priors) { + if (!p.pull_request && p.title.startsWith(titlePrefix)) { + try { + await github.rest.issues.update({ + owner: context.repo.owner, repo: context.repo.repo, + issue_number: p.number, state: 'closed', state_reason: 'completed', + }); + } catch (err) { core.warning(`could not close prior digest #${p.number}: ${err.message}`); } + } + } + const issue = await github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, - title: `${process.env.ORG} — workflow failures detected ${today}`, + title: `${titlePrefix} ${today}`, body: report, labels: ['health-check', 'automated-report'], }); @@ -186,8 +207,10 @@ jobs: - name: Migrate misaligned fleet-tracker issues uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_PAT_WORKFLOWS: ${{ secrets.GH_PAT_WORKFLOWS }} with: - github-token: ${{ secrets.GH_PAT_WORKFLOWS }} + github-token: ${{ env.GH_PAT_WORKFLOWS }} script: | // Find open fleet-tracker issues in .github-private that belong to other repos const allIssues = await github.paginate(github.rest.issues.listForRepo, { @@ -245,9 +268,10 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: ORG: ${{ env.ORG }} + GH_PAT_WORKFLOWS: ${{ secrets.GH_PAT_WORKFLOWS }} with: # Use GH_PAT_WORKFLOWS for cross-repo issues:write on target repos - github-token: ${{ secrets.GH_PAT_WORKFLOWS }} + github-token: ${{ env.GH_PAT_WORKFLOWS }} script: | const fs = require('fs'); @@ -353,6 +377,71 @@ jobs: } + - name: Close resolved fleet-tracker issues + # The tracking step stamps "Last updated ... on " on an issue every + # run its workflow is still above threshold. A workflow that recovers (≤10%) + # or is deleted stops being updated, so its stamp goes stale — but nothing + # closed the issue, so fleet-tracker issues accumulated forever (#711-class + # bot backlog). Close any fleet-tracker issue whose stamp is older than + # STALE_DAYS daily runs. Gated on fleet_high_failure.json existing so a + # failed scan cannot mass-close; staleness (per-issue stamp, not the current + # run) means a single bad-scan day never false-closes. + if: hashFiles('fleet_high_failure.json') != '' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + ORG: ${{ env.ORG }} + STALE_DAYS: '3' + GH_PAT_WORKFLOWS: ${{ secrets.GH_PAT_WORKFLOWS }} + with: + # Use GH_PAT_WORKFLOWS for cross-repo issues:write on target repos + github-token: ${{ env.GH_PAT_WORKFLOWS }} + script: | + const org = process.env.ORG; + const staleDays = parseInt(process.env.STALE_DAYS, 10); + const cutoffMs = staleDays * 24 * 60 * 60 * 1000; + const now = Date.now(); + + const found = await github.paginate(github.rest.search.issuesAndPullRequests, { + q: `org:${org} is:issue is:open label:fleet-tracker`, + per_page: 100, + }, response => response.data.items); + + let closed = 0, kept = 0, skipped = 0; + for (const issue of found) { + const rm = (issue.repository_url || '').match(/repos\/([^/]+)\/([^/]+)$/); + const dm = (issue.body || '').match(/updated by \[Fleet Monitor run\]\([^)]*\) on (\d{4}-\d{2}-\d{2})/); + // No repo match or no stamp → not a monitor-managed body; leave it alone. + if (!rm || !dm) { skipped++; continue; } + const [ , owner, repo ] = rm; + const ageMs = now - Date.parse(`${dm[1]}T00:00:00Z`); + if (ageMs <= cutoffMs) { kept++; continue; } // still fresh → workflow still failing + const days = Math.floor(ageMs / 86400000); + const body = [ + `✅ **Auto-resolved by Fleet Monitor.** This workflow's failure rate has`, + `been below the 10% threshold (or the workflow was removed) since ${dm[1]}`, + `(${days} days) — the monitor stopped flagging it. Closing. It will reopen`, + `automatically if the workflow crosses the threshold again.`, + ].join(' '); + try { + await github.rest.issues.createComment({ + owner, repo, issue_number: issue.number, body, + }); + } catch (err) { + core.warning(`comment failed on ${owner}/${repo}#${issue.number}: ${err.message}`); + } + try { + await github.rest.issues.update({ + owner, repo, issue_number: issue.number, + state: 'closed', state_reason: 'completed', + }); + closed++; + } catch (err) { + core.warning(`close failed on ${owner}/${repo}#${issue.number}: ${err.message}`); + } + } + core.notice(`fleet-tracker auto-close: closed=${closed} kept=${kept} skipped=${skipped} (stale>${staleDays}d)`); + + - name: Dispatch dev-lead for critical items if: env.HIGH_FAILURE_COUNT != '' && env.HIGH_FAILURE_COUNT != '0' env: