From 6eb97fdb5798e9bb181f2b246d3ce7ca78398794 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 5 Aug 2026 01:25:01 +0000 Subject: [PATCH 1/4] WIP: F1: Falsifier totality receipt + oncall router --- .github/workflows/falsifier-alert.yml | 148 ++++++++++++++++++++++++-- 1 file changed, 140 insertions(+), 8 deletions(-) diff --git a/.github/workflows/falsifier-alert.yml b/.github/workflows/falsifier-alert.yml index 8560fc67270..d41d1b2cf54 100644 --- a/.github/workflows/falsifier-alert.yml +++ b/.github/workflows/falsifier-alert.yml @@ -9,6 +9,21 @@ # issue (notifying watchers), sustained red heartbeats at most daily, and # transition-to-green closes it with a receipt. # +# The open issue describes the CURRENT failure, not the first one. Earlier this +# workflow wrote the body once at first red and thereafter only appended a +# throttled heartbeat comment, so the body — the thing a human actually reads — +# froze while the failure changed underneath it. Receipt: #7737 sat open for ~30h +# describing a BudgetExceeded kill on witness_committed_is_fixed_point that had +# stopped occurring at 2026-08-03T17:53, while the live failure was WitnessRed on +# five entirely different witnesses; recovering the real failing set meant +# downloading a two-hour run log and grepping it. An open issue reads as an +# accurate live description of the current state, so a stale one is worse than +# none. The body is therefore rewritten every red window (body edits do not +# notify, so this costs no noise), and the failing set plus its failure class is +# carried as a signature: a red that CHANGES CLASS is a different incident from a +# red that persists, and it bypasses the comment throttle rather than being +# suppressed by it. +# # It reports PER-COMPONENT state read from the floor's own receipt # (gunbc.floor_component_receipt, uploaded by the falsifier job as # floor-component-receipt), never inferred from the overall workflow conclusion. @@ -145,8 +160,70 @@ jobs: return lines.join('\n'); } + // ---- failing-set / failure-class signature ------------------------- + // The signature is what makes a red that CHANGED CLASS distinguishable + // from a red that merely persists. They are different incidents with + // different remedies (re-basis a dated ceiling vs. fix a witness vs. + // repair a runner), and the 20h heartbeat throttle below is keyed on it + // so a transition is never suppressed as "still red". + // + // It is derived from the receipt's own identities — component index, + // label, and the outcome tag the floor already emits — never from + // rendered prose. `refused` (a deliberate stop at a declared ceiling, + // which is where BudgetExceeded lands) and `failed` are distinct tags in + // the receipt, so a BudgetExceeded -> WitnessRed transition changes this + // string. An unreadable receipt gets its own signature carrying the + // cause: it is an unknown, never folded into any known failing set. + function signatureOf(r) { + if (!r.ok) return `receipt-unavailable:${r.cause}`; + const bad = Array.isArray(r.doc.unsuccessful_components) + ? r.doc.unsuccessful_components : []; + if (bad.length === 0) return 'no-unsuccessful-components'; + return bad + .map((c) => `${c.index}:${c.label}:${c.outcome}`) + .sort() + .join('|'); + } + + // Prior state rides in the issue body as a marker, so no external store + // is needed and the issue stays the single authority for its own state. + // A body with no marker (an issue opened by the previous version of this + // workflow) yields null — an explicit unknown, which compares unequal to + // every real signature, so the first run after this lands rewrites the + // stale body instead of silently keeping it. + const STATE_RE = //; + function readState(body) { + const m = STATE_RE.exec(body || ''); + if (!m) return { first_red: null, signature: null }; + try { + const s = JSON.parse(m[1]); + return { + first_red: typeof s.first_red === 'string' ? s.first_red : null, + signature: typeof s.signature === 'string' ? s.signature : null, + }; + } catch (e) { + return { first_red: null, signature: null }; + } + } + + // Owner routing is REFUSED, not guessed. gunbc.falsifier_lane derives a + // lane for a failing component but its owner roster is honestly empty, + // and minting a @mention from a string in this file would fabricate a + // routing fact that no carrier grounds — a second authority for + // ownership living in workflow YAML. + // 🟡 dissolve-on: the owner grounding carrier lands; this block then + // renders the routed owner (or that carrier's own typed refusal). + const ROUTING = [ + '**Owner routing: REFUSED — no grounding carrier.**', + '', + 'The failing component(s) above are named, but no declared roster maps a', + 'falsifier lane to an owner, so this alert routes to nobody rather than', + 'guessing. Treat the missing roster as its own obligation.', + ].join('\n'); + const receipt = readReceipt(); const report = componentReport(receipt); + const signature = signatureOf(receipt); let titleSuffix; if (!receipt.ok) { @@ -166,36 +243,91 @@ jobs: const existing = open[0]; if (conclusion === 'failure') { + const prior = readState(existing ? existing.body : null); + // The first-red line is a SNAPSHOT and is preserved verbatim across + // rewrites; "Current" is re-derived every window. Losing the former + // would discard when the incident started, keeping only the latter + // would leave the body describing a failure that no longer occurs — + // which is exactly the defect this fixes (#7737 sat for ~30h showing a + // BudgetExceeded story while the actual failure was five red witnesses). + const firstRed = prior.first_red || runLine; + const state = JSON.stringify({ first_red: firstRed, signature }); const body = [ - '@briansrls — the scheduled affected-set falsifier just went RED.', + '@briansrls — the scheduled affected-set falsifier is RED.', + '', + `First red: ${firstRed}`, + `Current: ${runLine}`, '', - `First red: ${runLine}`, + '## Current failing set', '', report, '', - 'This issue auto-closes when a scheduled run goes green; sustained', - 'red heartbeats at most daily.', + '## Routing', + '', + ROUTING, + '', + 'This issue auto-closes when a scheduled run goes green. The body above', + 'is REWRITTEN every red window, so it always describes the current', + 'failure rather than the first one. Comments are throttled to at most', + 'every 20h while the failing set and its failure class are unchanged; a', + 'change in either is a different incident and comments immediately.', + '', + ``, ].join('\n'); if (!existing) { await github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, title, labels: [label], body, }); - core.notice('falsifier alert issue opened'); + core.notice(`falsifier alert issue opened — signature ${signature}`); } else { - // heartbeat at most every 20h while red + // The body and title are rewritten on EVERY red window. Editing an + // issue body sends no notification, so this costs no noise and is + // deliberately not rate-limited: a stale-but-open issue reads as an + // accurate live description of the current failure, which is worse + // than no description at all. + await github.rest.issues.update({ + owner: context.repo.owner, repo: context.repo.repo, + issue_number: existing.number, title, body, + }); + + // Comments DO notify, so they stay throttled — except across a + // transition, which the throttle must not swallow. + const changed = prior.signature !== signature; const { data: comments } = await github.rest.issues.listComments({ owner: context.repo.owner, repo: context.repo.repo, issue_number: existing.number, per_page: 100, }); const last = comments[comments.length - 1]; const lastTs = new Date(last ? last.created_at : existing.created_at).getTime(); - if (Date.now() - lastTs > 20 * 3600 * 1000) { + const heartbeatDue = Date.now() - lastTs > 20 * 3600 * 1000; + if (changed) { + const priorLine = prior.signature === null + ? 'not recorded (issue predates signature tracking)' + : `\`${prior.signature}\``; + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, + issue_number: existing.number, + body: [ + `**The failing set changed** — this is a different incident from the one first reported.`, + '', + `Previous: ${priorLine}`, + `Now: \`${signature}\``, + '', + `Latest: ${runLine}`, + '', + report, + ].join('\n'), + }); + core.notice(`falsifier failing set changed: ${prior.signature} -> ${signature}`); + } else if (heartbeatDue) { await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: existing.number, - body: `Still red. Latest: ${runLine}\n\n${report}`, + body: `Still red, unchanged failing set. Latest: ${runLine}\n\n${report}`, }); + } else { + core.info(`red, signature unchanged (${signature}) — body refreshed, comment throttled`); } } } else if (existing) { From 00c73cbcd344ce6d10fa124eb5f0c364a2d3198e Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 5 Aug 2026 01:36:00 +0000 Subject: [PATCH 2/4] Recover the true first-red line instead of stamping the current run as First red MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by executing readState against the REAL #7737 body rather than a fixture: its body has no state marker, so first_red parsed to null and the rewrite would have labelled the current run 'First red' — asserting something false in the very body this change exists to keep honest. The legacy rendered 'First red: ...' line is the only record of when an incident started, so it is now recovered from it. If neither the marker nor the legacy line is present on an existing issue, the first red is reported as not recorded; it is never back-filled with the current run. A genuinely new issue still records this run, which is correct. Verified by execution against #7737's actual body (recovers run 30809682025) and by mutation: deleting the recovery arm reds 2 controls. --- .github/workflows/falsifier-alert.yml | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/falsifier-alert.yml b/.github/workflows/falsifier-alert.yml index d41d1b2cf54..d7d649d460d 100644 --- a/.github/workflows/falsifier-alert.yml +++ b/.github/workflows/falsifier-alert.yml @@ -192,9 +192,21 @@ jobs: // every real signature, so the first run after this lands rewrites the // stale body instead of silently keeping it. const STATE_RE = //; + // Legacy recovery: issues opened before the marker existed still carry a + // rendered "First red: ..." line, and that line is the ONLY record of when + // the incident started. Recovering it is not cosmetic — without it the + // rewrite below would stamp the CURRENT run as "First red", asserting + // something false in the body, which is the same class of defect this + // change exists to remove. If neither the marker nor the legacy line is + // present, the first red is reported as not recorded; it is never guessed. + const LEGACY_FIRST_RED_RE = /^First red: (.+)$/m; function readState(body) { - const m = STATE_RE.exec(body || ''); - if (!m) return { first_red: null, signature: null }; + const text = body || ''; + const m = STATE_RE.exec(text); + if (!m) { + const legacy = LEGACY_FIRST_RED_RE.exec(text); + return { first_red: legacy ? legacy[1].trim() : null, signature: null }; + } try { const s = JSON.parse(m[1]); return { @@ -202,7 +214,8 @@ jobs: signature: typeof s.signature === 'string' ? s.signature : null, }; } catch (e) { - return { first_red: null, signature: null }; + const legacy = LEGACY_FIRST_RED_RE.exec(text); + return { first_red: legacy ? legacy[1].trim() : null, signature: null }; } } @@ -250,7 +263,12 @@ jobs: // would leave the body describing a failure that no longer occurs — // which is exactly the defect this fixes (#7737 sat for ~30h showing a // BudgetExceeded story while the actual failure was five red witnesses). - const firstRed = prior.first_red || runLine; + // For a NEW issue this run genuinely is the first red. For an existing + // one it is not, so an unrecoverable first red is reported as unknown + // rather than back-filled with the current run. + const firstRed = existing + ? (prior.first_red || 'not recorded (issue predates state tracking)') + : runLine; const state = JSON.stringify({ first_red: firstRed, signature }); const body = [ '@briansrls — the scheduled affected-set falsifier is RED.', From e92f2eb22855bbe75c6901c3f5296f056982286f Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 5 Aug 2026 01:40:49 +0000 Subject: [PATCH 3/4] A cold control that DID NOT RUN is not one that FAILED MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit royal-raven's #7818 makes an enrolled-but-unreached component keep its planned identity, so the affected-set cold control now resolves to present/unverified with a Skipped outcome instead of collapsing to state=absent. This arm labelled every present-but-unverified control 'RED', which would then report a control that never ran as a failing one — a state-space conflation with different remedies (restore the enrollment vs fix the control). Both arms still say selection runs UNVERIFIED, so fail-closed is unchanged; only the label distinguishes them. Also gives 'not_concluded' a phrase instead of rendering the raw tag. Mutation control: forcing didNotRun=false reds the discriminating assertion. --- .github/workflows/falsifier-alert.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/falsifier-alert.yml b/.github/workflows/falsifier-alert.yml index d7d649d460d..e1fd05086f0 100644 --- a/.github/workflows/falsifier-alert.yml +++ b/.github/workflows/falsifier-alert.yml @@ -106,6 +106,8 @@ jobs: if (c.outcome === 'failed') return `Failed — ${c.error}`; if (c.outcome === 'skipped') return `Skipped — ${c.reason}`; if (c.outcome === 'done') return 'Done'; + // The receipt's own arm for a component that reached no conclusion. + if (c.outcome === 'not_concluded') return 'reached no conclusion'; return String(c.outcome); } @@ -137,8 +139,18 @@ jobs: lines.push(`**Affected-set cold control: GREEN** — concluded Done over ${n} witnesses.`); lines.push('Per-PR affected-set selection IS verified by this run.'); } else if (ctl.state === 'present') { + // A control that DID NOT RUN is not a control that FAILED. Both leave + // selection unverified — that stays fail-closed either way — but they + // have different remedies (restore the enrollment vs. fix the control), + // so labelling a Skipped control "RED" is a state-space conflation of + // exactly the kind this receipt exists to prevent. This arm sees Skipped + // once unreached components keep their planned identity (#7818): they + // resolve to present/unverified rather than collapsing to absent. const c = ctl.component || {}; - lines.push(`**Affected-set cold control: RED** — ${describeOutcome(c)}.`); + const didNotRun = c.outcome === 'skipped' || c.outcome === 'not_concluded'; + lines.push(didNotRun + ? `**Affected-set cold control: DID NOT RUN** — ${describeOutcome(c)}.` + : `**Affected-set cold control: RED** — ${describeOutcome(c)}.`); lines.push('Per-PR affected-set selection runs UNVERIFIED while this stands.'); } else { lines.push(`**Affected-set cold control: ${String(ctl.state || 'unknown').toUpperCase()}** — ${ctl.cause || 'no cause recorded'}.`); From 4aa7489c804b25f8f4421ab3e7ee0c217c8048d1 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 5 Aug 2026 02:20:39 +0000 Subject: [PATCH 4/4] Correct the header's account of the #7737 defect: alerting worked, the body lied MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit still-bat-561 verified the alert MECHANISM is not broken — #7737's 2026-08-04T12:31 heartbeat comment does name the current failing set, at the promised at-most-daily cadence. The previous wording said recovering the real failing set meant downloading a run log and grepping it, which overstated the defect: the set was in the issue, in a comment. What is actually wrong is narrower and worth stating exactly, since a receipt that overstates its own incident is the same class of defect as the stale body: the title and body — what a reader meets first — describe a resolved failure, the current one sits in a comment up to 20h old below the fold, and the class transition announced nothing because BudgetExceeded -> WitnessRed was absorbed into an unchanged Still red. No behavioural change; this corrects the justification only. --- .github/workflows/falsifier-alert.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/falsifier-alert.yml b/.github/workflows/falsifier-alert.yml index e1fd05086f0..a7d65c85bed 100644 --- a/.github/workflows/falsifier-alert.yml +++ b/.github/workflows/falsifier-alert.yml @@ -13,11 +13,17 @@ # workflow wrote the body once at first red and thereafter only appended a # throttled heartbeat comment, so the body — the thing a human actually reads — # froze while the failure changed underneath it. Receipt: #7737 sat open for ~30h -# describing a BudgetExceeded kill on witness_committed_is_fixed_point that had -# stopped occurring at 2026-08-03T17:53, while the live failure was WitnessRed on -# five entirely different witnesses; recovering the real failing set meant -# downloading a two-hour run log and grepping it. An open issue reads as an -# accurate live description of the current state, so a stale one is worse than +# with a body describing a BudgetExceeded kill on witness_committed_is_fixed_point +# that had stopped occurring at 2026-08-03T17:53, while the live failure was +# WitnessRed on entirely different witnesses. Precisely: the alerting MECHANISM was +# working — the 2026-08-04T12:31 heartbeat comment does name the current failing set +# — so this is not missed alerting. What failed is that the two things a reader +# meets first, the title and the body, described a resolved failure, while the +# current one was only in a comment that may be up to 20h old and sits below the +# fold. Worse, the class transition itself never announced anything: BudgetExceeded +# -> WitnessRed is a different incident with a different remedy, and it was absorbed +# into an unchanged "Still red". An open issue reads as an accurate live description +# of the current state, so a stale one is worse than # none. The body is therefore rewritten every red window (body edits do not # notify, so this costs no noise), and the failing set plus its failure class is # carried as a signature: a red that CHANGES CLASS is a different incident from a