From fb1ac5b76cf3e29eef93f86efb3ffea8a170b92c Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 6 Aug 2026 12:00:15 +0100 Subject: [PATCH 01/18] fix: keep builder circuit breaker state when window has no blocks --- .../beacon-node/src/chain/builderCircuitBreaker.ts | 7 +++++-- .../test/unit/chain/builderCircuitBreaker.test.ts | 13 ++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/beacon-node/src/chain/builderCircuitBreaker.ts b/packages/beacon-node/src/chain/builderCircuitBreaker.ts index ba09c90e66cc..3dbfb7818ae2 100644 --- a/packages/beacon-node/src/chain/builderCircuitBreaker.ts +++ b/packages/beacon-node/src/chain/builderCircuitBreaker.ts @@ -57,8 +57,11 @@ export class BuilderCircuitBreaker { const faults = blocksPresent - payloadsRevealed; const wasActive = this.active; - // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates - this.active = faults * this.faultInspectionWindow > this.allowedFaults * blocksPresent; + // Keep the previous state if the window has no blocks, there is no data to assess builder health + if (blocksPresent > 0) { + // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates + this.active = faults * this.faultInspectionWindow > this.allowedFaults * blocksPresent; + } this.modules.metrics?.builderCircuitBreaker.active.set(this.active ? 1 : 0); this.modules.metrics?.builderCircuitBreaker.faults.set(faults); diff --git a/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts b/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts index cfde576e7f44..49e6a5c1da6c 100644 --- a/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts +++ b/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts @@ -21,7 +21,7 @@ describe("BuilderCircuitBreaker", () => { } const testCases: [string, {blocksPresent: number; payloadsRevealed: number}, boolean][] = [ - ["empty window", {blocksPresent: 0, payloadsRevealed: 0}, false], + ["empty window keeps initial state", {blocksPresent: 0, payloadsRevealed: 0}, false], ["full window, no faults", {blocksPresent: 32, payloadsRevealed: 32}, false], ["full window, faults at budget", {blocksPresent: 32, payloadsRevealed: 24}, false], ["full window, faults above budget", {blocksPresent: 32, payloadsRevealed: 23}, true], @@ -43,6 +43,17 @@ describe("BuilderCircuitBreaker", () => { expect(getPayloadRevealCounts).toHaveBeenCalledWith(100 - faultInspectionWindow, 99); }); + it("keeps previous state while window has no blocks", () => { + const {breaker, getPayloadRevealCounts} = setup({blocksPresent: 8, payloadsRevealed: 0}); + expect(breaker.isActive(100)).toBe(true); + + getPayloadRevealCounts.mockReturnValue({blocksPresent: 0, payloadsRevealed: 0}); + expect(breaker.isActive(101)).toBe(true); + + getPayloadRevealCounts.mockReturnValue({blocksPresent: 8, payloadsRevealed: 8}); + expect(breaker.isActive(102)).toBe(false); + }); + it("only updates once per slot", () => { const {breaker, getPayloadRevealCounts} = setup({blocksPresent: 32, payloadsRevealed: 32}); expect(breaker.isActive(100)).toBe(false); From d6c94c03a7ba5ae0a3ad20ed3be499f4121378fb Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 6 Aug 2026 12:03:41 +0100 Subject: [PATCH 02/18] add clock slot to log --- packages/beacon-node/src/chain/builderCircuitBreaker.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/beacon-node/src/chain/builderCircuitBreaker.ts b/packages/beacon-node/src/chain/builderCircuitBreaker.ts index 3dbfb7818ae2..597a39f49586 100644 --- a/packages/beacon-node/src/chain/builderCircuitBreaker.ts +++ b/packages/beacon-node/src/chain/builderCircuitBreaker.ts @@ -69,6 +69,7 @@ export class BuilderCircuitBreaker { this.modules.metrics?.builderCircuitBreaker.payloadsRevealed.set(payloadsRevealed); const logCtx = { + clockSlot, blocksPresent, faults, faultInspectionWindow: this.faultInspectionWindow, From 362b2803cd7042fe363bab3fca34e5613a223be1 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 6 Aug 2026 12:21:41 +0100 Subject: [PATCH 03/18] fix: exclude anchor node from payload reveal counts --- .../fork-choice/src/protoArray/protoArray.ts | 4 ++++ .../test/unit/protoArray/gloas.test.ts | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index b234f70d3b24..2641d449a3cc 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -704,6 +704,10 @@ export class ProtoArray { if (node.slot < fromSlot || node.slot > toSlot || node.payloadStatus !== PayloadStatus.PENDING) { continue; } + // Skip anchor nodes (no parent), they are seeded without payload data and cannot be assessed + if (node.parent === undefined) { + continue; + } blocksPresent++; if (this.hasPayload(node.blockRoot)) { payloadsRevealed++; diff --git a/packages/fork-choice/test/unit/protoArray/gloas.test.ts b/packages/fork-choice/test/unit/protoArray/gloas.test.ts index f99537953362..285981795fe0 100644 --- a/packages/fork-choice/test/unit/protoArray/gloas.test.ts +++ b/packages/fork-choice/test/unit/protoArray/gloas.test.ts @@ -226,6 +226,22 @@ describe("Gloas Fork Choice", () => { payloadsRevealed: 0, }); }); + + it("does not count the anchor block seeded at initialization", () => { + const currentSlot = gloasForkSlot + 1; + // Gloas anchor is seeded as PENDING without payload data, as after checkpoint sync or gloas genesis + const protoArray = ProtoArray.initialize( + createTestBlock(gloasForkSlot, genesisRoot, "0x00", "0x00"), + currentSlot + ); + + protoArray.onBlock(createTestBlock(gloasForkSlot + 1, "0x02", genesisRoot, genesisRoot), currentSlot, null); + + expect(protoArray.getPayloadRevealCounts(gloasForkSlot, currentSlot)).toEqual({ + blocksPresent: 1, + payloadsRevealed: 0, + }); + }); }); describe("Pre-Gloas (Fulu) behavior", () => { From c766581a348727e79e29f76a5ca772fe81caad1a Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 6 Aug 2026 12:21:42 +0100 Subject: [PATCH 04/18] refactor: clarify builder circuit breaker params and docs --- .../beacon-node/src/execution/builder/http.ts | 20 +++++++++---------- .../src/options/beaconNodeOptions/builder.ts | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/beacon-node/src/execution/builder/http.ts b/packages/beacon-node/src/execution/builder/http.ts index 78d029bdf584..7a60312ad15f 100644 --- a/packages/beacon-node/src/execution/builder/http.ts +++ b/packages/beacon-node/src/execution/builder/http.ts @@ -71,15 +71,18 @@ export class NoBidReceived extends Error { } /** - * Beacon clients select randomized values from the following ranges when initializing - * the circuit breaker (so at boot time and once for each unique boot). + * The fault inspection window is randomized when initializing the circuit breaker (so at + * boot time and once for each unique boot), the fault budget is derived from it: * - * ALLOWED_FAULTS: between 1 and SLOTS_PER_EPOCH // 4 * FAULT_INSPECTION_WINDOW: between SLOTS_PER_EPOCH and 2 * SLOTS_PER_EPOCH + * ALLOWED_FAULTS: FAULT_INSPECTION_WINDOW // 4 * - * The values are randomized per node so builders cannot predict when a given proposer will - * fall back to local blocks. With fixed thresholds a builder could withhold payloads right up - * to the limit without ever tripping the breaker. + * e.g. on mainnet SLOTS_PER_EPOCH is 32, so FAULT_INSPECTION_WINDOW is between 32 and 63, + * and a window of 40 results in ALLOWED_FAULTS = 10. + * + * The tolerated fault rate is ~25% on every node, but the randomized window keeps the exact + * slots under inspection unpredictable per node, so a builder cannot tell when past faults + * age out of a given proposer's window and time withholding around trip or recovery points. */ export function getFaultInspectionParams(opts: {faultInspectionWindow?: number; allowedFaults?: number}): { faultInspectionWindow: number; @@ -90,10 +93,7 @@ export function getFaultInspectionParams(opts: {faultInspectionWindow?: number; SLOTS_PER_EPOCH ); // allowedFaults should be < faultInspectionWindow, limiting them to faultInspectionWindow/4 - const allowedFaults = Math.min( - opts.allowedFaults ?? Math.floor(faultInspectionWindow / 4), - Math.floor(faultInspectionWindow / 4) - ); + const allowedFaults = Math.min(opts.allowedFaults ?? Infinity, Math.floor(faultInspectionWindow / 4)); return {faultInspectionWindow, allowedFaults}; } diff --git a/packages/cli/src/options/beaconNodeOptions/builder.ts b/packages/cli/src/options/beaconNodeOptions/builder.ts index 25f7d09efc58..efc5bd4fbcf2 100644 --- a/packages/cli/src/options/beaconNodeOptions/builder.ts +++ b/packages/cli/src/options/beaconNodeOptions/builder.ts @@ -62,7 +62,7 @@ export const options: CliCommandOptions = { "builder.allowedFaults": { type: "number", description: - "Number of missed slots (pre-gloas) or unrevealed payloads (post-gloas) allowed in the `faultInspectionWindow` for builder circuit", + "Number of missed slots allowed within `faultInspectionWindow` before ignoring the external builder (pre-gloas). Post-gloas, sets the tolerated rate of unrevealed payloads, defined as `allowedFaults` out of `faultInspectionWindow` and applied to blocks observed in the window", group: "builder", }, }; From 15dba562bd483032ef735702400314c146a2410c Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Aug 2026 19:15:06 +0100 Subject: [PATCH 05/18] clarify skipped parentless proto-array nodes --- packages/fork-choice/src/protoArray/protoArray.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index 2641d449a3cc..c0b3ed061dc5 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -704,7 +704,8 @@ export class ProtoArray { if (node.slot < fromSlot || node.slot > toSlot || node.payloadStatus !== PayloadStatus.PENDING) { continue; } - // Skip anchor nodes (no parent), they are seeded without payload data and cannot be assessed + // Skip roots without a retained parent. This includes the anchor, which is seeded without + // payload data, and stale branch roots disconnected by pruning. if (node.parent === undefined) { continue; } From 11711132f037888b74253371ca42b245d0c02209 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Aug 2026 19:15:17 +0100 Subject: [PATCH 06/18] require a minimum sample before breaker recovery --- .../beacon-node/src/chain/builderCircuitBreaker.ts | 13 +++++++++---- .../test/unit/chain/builderCircuitBreaker.test.ts | 12 ++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/packages/beacon-node/src/chain/builderCircuitBreaker.ts b/packages/beacon-node/src/chain/builderCircuitBreaker.ts index 597a39f49586..d1d22fa380f8 100644 --- a/packages/beacon-node/src/chain/builderCircuitBreaker.ts +++ b/packages/beacon-node/src/chain/builderCircuitBreaker.ts @@ -15,6 +15,8 @@ export type BuilderCircuitBreakerModules = { metrics: Metrics | null; }; +const MIN_BLOCKS_TO_DEACTIVATE = 4; + /** * Post-gloas circuit breaker for builder bids. The beacon block is produced by the proposer * regardless of bid source, so missed blocks are not a useful builder health signal. Instead @@ -57,10 +59,13 @@ export class BuilderCircuitBreaker { const faults = blocksPresent - payloadsRevealed; const wasActive = this.active; - // Keep the previous state if the window has no blocks, there is no data to assess builder health - if (blocksPresent > 0) { - // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates - this.active = faults * this.faultInspectionWindow > this.allowedFaults * blocksPresent; + // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates + const exceedsFaultBudget = faults * this.faultInspectionWindow > this.allowedFaults * blocksPresent; + if (exceedsFaultBudget) { + this.active = true; + } else if (blocksPresent >= MIN_BLOCKS_TO_DEACTIVATE) { + // Require a small healthy sample before accepting builder bids again + this.active = false; } this.modules.metrics?.builderCircuitBreaker.active.set(this.active ? 1 : 0); diff --git a/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts b/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts index 49e6a5c1da6c..3f7c53b81fae 100644 --- a/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts +++ b/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts @@ -27,6 +27,7 @@ describe("BuilderCircuitBreaker", () => { ["full window, faults above budget", {blocksPresent: 32, payloadsRevealed: 23}, true], ["sparse window, faults within scaled budget", {blocksPresent: 8, payloadsRevealed: 6}, false], ["sparse window, faults above scaled budget", {blocksPresent: 8, payloadsRevealed: 5}, true], + ["single unrevealed payload", {blocksPresent: 1, payloadsRevealed: 0}, true], ["sparse window, all payloads unrevealed", {blocksPresent: 4, payloadsRevealed: 0}, true], ]; @@ -43,15 +44,18 @@ describe("BuilderCircuitBreaker", () => { expect(getPayloadRevealCounts).toHaveBeenCalledWith(100 - faultInspectionWindow, 99); }); - it("keeps previous state while window has no blocks", () => { - const {breaker, getPayloadRevealCounts} = setup({blocksPresent: 8, payloadsRevealed: 0}); + it("requires a minimum sample to deactivate", () => { + const {breaker, getPayloadRevealCounts} = setup({blocksPresent: 1, payloadsRevealed: 0}); expect(breaker.isActive(100)).toBe(true); getPayloadRevealCounts.mockReturnValue({blocksPresent: 0, payloadsRevealed: 0}); expect(breaker.isActive(101)).toBe(true); - getPayloadRevealCounts.mockReturnValue({blocksPresent: 8, payloadsRevealed: 8}); - expect(breaker.isActive(102)).toBe(false); + getPayloadRevealCounts.mockReturnValue({blocksPresent: 3, payloadsRevealed: 3}); + expect(breaker.isActive(102)).toBe(true); + + getPayloadRevealCounts.mockReturnValue({blocksPresent: 4, payloadsRevealed: 3}); + expect(breaker.isActive(103)).toBe(false); }); it("only updates once per slot", () => { From 6e797759903683cb276ee03d683bd11ab223e453 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Aug 2026 19:15:30 +0100 Subject: [PATCH 07/18] clarify allowed faults option semantics --- packages/beacon-node/src/chain/options.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index 844aa8e8baa6..560bc1de68c2 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -51,7 +51,10 @@ export type IChainOptions = BlockProcessOpts & nativeStateView?: boolean; /** Builder circuit breaker fault inspection window in slots */ faultInspectionWindow?: number; - /** Allowed unrevealed payloads within the fault inspection window */ + /** + * Allowed missed slots pre-Gloas. Post-Gloas, sets the tolerated unrevealed payload rate as + * allowedFaults / faultInspectionWindow. + */ allowedFaults?: number; }; From 00db54a3f8e1c012b3774515d07243289afe57e9 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Aug 2026 19:15:17 +0100 Subject: [PATCH 08/18] document minimum breaker recovery sample --- packages/beacon-node/src/chain/builderCircuitBreaker.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/beacon-node/src/chain/builderCircuitBreaker.ts b/packages/beacon-node/src/chain/builderCircuitBreaker.ts index d1d22fa380f8..56792ec75e9e 100644 --- a/packages/beacon-node/src/chain/builderCircuitBreaker.ts +++ b/packages/beacon-node/src/chain/builderCircuitBreaker.ts @@ -15,6 +15,7 @@ export type BuilderCircuitBreakerModules = { metrics: Metrics | null; }; +/** Four observations is the minimum useful recovery sample for the default ~25% fault budget */ const MIN_BLOCKS_TO_DEACTIVATE = 4; /** From 2cdb351dc16325216455a1d8da49932d3cb2ad54 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Aug 2026 19:36:15 +0100 Subject: [PATCH 09/18] document allowed faults behavior --- packages/beacon-node/src/chain/options.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index 560bc1de68c2..580d54273bc5 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -52,8 +52,9 @@ export type IChainOptions = BlockProcessOpts & /** Builder circuit breaker fault inspection window in slots */ faultInspectionWindow?: number; /** - * Allowed missed slots pre-Gloas. Post-Gloas, sets the tolerated unrevealed payload rate as - * allowedFaults / faultInspectionWindow. + * Missed slots tolerated within `faultInspectionWindow` pre-Gloas. Post-Gloas, activates when + * `unrevealedPayloads * faultInspectionWindow > allowedFaults * blocksPresent` and deactivates + * after at least four blocks within budget. Defaults to and is capped at `faultInspectionWindow / 4`. */ allowedFaults?: number; }; From ed70d6bb2c6c77be17db550bec2cc37eb78f171d Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Aug 2026 19:43:23 +0100 Subject: [PATCH 10/18] simplify allowed faults documentation --- packages/beacon-node/src/chain/options.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index 580d54273bc5..1ae59f10cab8 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -52,9 +52,8 @@ export type IChainOptions = BlockProcessOpts & /** Builder circuit breaker fault inspection window in slots */ faultInspectionWindow?: number; /** - * Missed slots tolerated within `faultInspectionWindow` pre-Gloas. Post-Gloas, activates when - * `unrevealedPayloads * faultInspectionWindow > allowedFaults * blocksPresent` and deactivates - * after at least four blocks within budget. Defaults to and is capped at `faultInspectionWindow / 4`. + * Missed slots allowed pre-Gloas; post-Gloas, unrevealed payloads allowed per + * `faultInspectionWindow` observed blocks. */ allowedFaults?: number; }; From fd0a425a11615da583cc910ae785f7066cb45c22 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Aug 2026 19:48:47 +0100 Subject: [PATCH 11/18] correct allowed faults documentation scope --- packages/beacon-node/src/chain/options.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index 1ae59f10cab8..ea9bab796d30 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -51,10 +51,7 @@ export type IChainOptions = BlockProcessOpts & nativeStateView?: boolean; /** Builder circuit breaker fault inspection window in slots */ faultInspectionWindow?: number; - /** - * Missed slots allowed pre-Gloas; post-Gloas, unrevealed payloads allowed per - * `faultInspectionWindow` observed blocks. - */ + /** Unrevealed payloads allowed per `faultInspectionWindow` observed blocks post-Gloas */ allowedFaults?: number; }; From 69375d6b5fedebb49f97fbaf3eb1880925c08e50 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Aug 2026 20:14:36 +0100 Subject: [PATCH 12/18] simplify allowed faults documentation --- packages/beacon-node/src/chain/options.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index ea9bab796d30..8a1267126601 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -51,7 +51,7 @@ export type IChainOptions = BlockProcessOpts & nativeStateView?: boolean; /** Builder circuit breaker fault inspection window in slots */ faultInspectionWindow?: number; - /** Unrevealed payloads allowed per `faultInspectionWindow` observed blocks post-Gloas */ + /** Unrevealed payloads allowed per `faultInspectionWindow` observed blocks */ allowedFaults?: number; }; From 458835fe10ac21927169e48743c56d090258eae4 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Aug 2026 20:37:15 +0100 Subject: [PATCH 13/18] clarify default fault budget documentation --- packages/beacon-node/src/execution/builder/http.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/src/execution/builder/http.ts b/packages/beacon-node/src/execution/builder/http.ts index 7a60312ad15f..9975a0c32cf0 100644 --- a/packages/beacon-node/src/execution/builder/http.ts +++ b/packages/beacon-node/src/execution/builder/http.ts @@ -80,7 +80,7 @@ export class NoBidReceived extends Error { * e.g. on mainnet SLOTS_PER_EPOCH is 32, so FAULT_INSPECTION_WINDOW is between 32 and 63, * and a window of 40 results in ALLOWED_FAULTS = 10. * - * The tolerated fault rate is ~25% on every node, but the randomized window keeps the exact + * The default tolerated fault rate is ~25%, but the randomized window keeps the exact * slots under inspection unpredictable per node, so a builder cannot tell when past faults * age out of a given proposer's window and time withholding around trip or recovery points. */ From 601db2fff13d79b2502e834be7e63aa734686aa6 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Aug 2026 21:02:59 +0100 Subject: [PATCH 14/18] tweak comments --- .../beacon-node/src/chain/builderCircuitBreaker.ts | 7 ++++--- packages/beacon-node/src/execution/builder/http.ts | 12 ++++++------ packages/fork-choice/src/forkChoice/interface.ts | 4 ++-- packages/fork-choice/src/protoArray/protoArray.ts | 4 ++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/packages/beacon-node/src/chain/builderCircuitBreaker.ts b/packages/beacon-node/src/chain/builderCircuitBreaker.ts index 56792ec75e9e..b632fc182088 100644 --- a/packages/beacon-node/src/chain/builderCircuitBreaker.ts +++ b/packages/beacon-node/src/chain/builderCircuitBreaker.ts @@ -21,8 +21,9 @@ const MIN_BLOCKS_TO_DEACTIVATE = 4; /** * Post-gloas circuit breaker for builder bids. The beacon block is produced by the proposer * regardless of bid source, so missed blocks are not a useful builder health signal. Instead - * count blocks whose payload was never revealed and stop selecting builder bids while the - * non-reveal rate in the fault inspection window is too high. + * count blocks whose payload was never revealed. Activate when the non-reveal rate exceeds the + * fault budget, and resume selecting builder bids only when the observed blocks are within budget + * and meet the minimum recovery sample size. */ export class BuilderCircuitBreaker { readonly faultInspectionWindow: number; @@ -65,7 +66,7 @@ export class BuilderCircuitBreaker { if (exceedsFaultBudget) { this.active = true; } else if (blocksPresent >= MIN_BLOCKS_TO_DEACTIVATE) { - // Require a small healthy sample before accepting builder bids again + // Require a minimum sample within the fault budget before accepting builder bids again this.active = false; } diff --git a/packages/beacon-node/src/execution/builder/http.ts b/packages/beacon-node/src/execution/builder/http.ts index 9975a0c32cf0..af5bf9e8576e 100644 --- a/packages/beacon-node/src/execution/builder/http.ts +++ b/packages/beacon-node/src/execution/builder/http.ts @@ -71,18 +71,18 @@ export class NoBidReceived extends Error { } /** - * The fault inspection window is randomized when initializing the circuit breaker (so at - * boot time and once for each unique boot), the fault budget is derived from it: + * Default circuit breaker parameters: * - * FAULT_INSPECTION_WINDOW: between SLOTS_PER_EPOCH and 2 * SLOTS_PER_EPOCH + * SLOTS_PER_EPOCH <= FAULT_INSPECTION_WINDOW < 2 * SLOTS_PER_EPOCH (randomized at initialization) * ALLOWED_FAULTS: FAULT_INSPECTION_WINDOW // 4 * * e.g. on mainnet SLOTS_PER_EPOCH is 32, so FAULT_INSPECTION_WINDOW is between 32 and 63, * and a window of 40 results in ALLOWED_FAULTS = 10. * - * The default tolerated fault rate is ~25%, but the randomized window keeps the exact - * slots under inspection unpredictable per node, so a builder cannot tell when past faults - * age out of a given proposer's window and time withholding around trip or recovery points. + * The randomized default window keeps the exact slots under inspection unpredictable per node, + * so a builder cannot tell when past faults age out of a given proposer's window and time + * withholding around trip or recovery points. Explicitly configured windows are clamped to at + * least SLOTS_PER_EPOCH, and configured allowedFaults is capped at the default ~25% budget. */ export function getFaultInspectionParams(opts: {faultInspectionWindow?: number; allowedFaults?: number}): { faultInspectionWindow: number; diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index 4a1b26a37d5f..fa8ed812c80c 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -247,8 +247,8 @@ export interface IForkChoice { hasPayloadHexUnsafe(blockRoot: RootHex): boolean; getSlotsPresent(windowStart: number): number; /** - * Count gloas blocks with fromSlot <= slot <= toSlot and how many of them have a revealed - * payload (FULL variant exists). Used by the builder circuit breaker. + * Count Gloas blocks with a retained parent and fromSlot <= slot <= toSlot, and how many of them + * have a revealed payload (FULL variant exists). Used by the builder circuit breaker. */ getPayloadRevealCounts(fromSlot: Slot, toSlot: Slot): {blocksPresent: number; payloadsRevealed: number}; getPTCVotes(blockRootHex: RootHex): (boolean | null)[] | null; diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index c0b3ed061dc5..e0e6baa44307 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -691,8 +691,8 @@ export class ProtoArray { } /** - * Count gloas blocks with fromSlot <= slot <= toSlot and how many of them have a revealed - * payload (FULL variant exists). Used by the builder circuit breaker. + * Count Gloas blocks with a retained parent and fromSlot <= slot <= toSlot, and how many of them + * have a revealed payload (FULL variant exists). Used by the builder circuit breaker. */ getPayloadRevealCounts(fromSlot: Slot, toSlot: Slot): {blocksPresent: number; payloadsRevealed: number} { let blocksPresent = 0; From bbce5926ec63fe76d3e75ac178feeb4970796b24 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Aug 2026 21:29:20 +0100 Subject: [PATCH 15/18] exclude genesis from payload reveal counts --- .../fork-choice/src/forkChoice/interface.ts | 4 +-- .../fork-choice/src/protoArray/protoArray.ts | 11 +++--- .../test/unit/protoArray/gloas.test.ts | 36 ++++++++++++++----- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index fa8ed812c80c..2c78375cc79d 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -247,8 +247,8 @@ export interface IForkChoice { hasPayloadHexUnsafe(blockRoot: RootHex): boolean; getSlotsPresent(windowStart: number): number; /** - * Count Gloas blocks with a retained parent and fromSlot <= slot <= toSlot, and how many of them - * have a revealed payload (FULL variant exists). Used by the builder circuit breaker. + * Count Gloas blocks with fromSlot <= slot <= toSlot, and how many of them have a revealed payload + * (FULL variant exists). Used by the builder circuit breaker. */ getPayloadRevealCounts(fromSlot: Slot, toSlot: Slot): {blocksPresent: number; payloadsRevealed: number}; getPTCVotes(blockRootHex: RootHex): (boolean | null)[] | null; diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index e0e6baa44307..012f0fad5569 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -1,5 +1,5 @@ import {BitArray} from "@chainsafe/ssz"; -import {GENESIS_EPOCH, PTC_SIZE} from "@lodestar/params"; +import {GENESIS_EPOCH, GENESIS_SLOT, PTC_SIZE} from "@lodestar/params"; import {DataAvailabilityStatus, computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Epoch, RootHex, Slot} from "@lodestar/types"; import {bitCount, toRootHex} from "@lodestar/utils"; @@ -691,8 +691,8 @@ export class ProtoArray { } /** - * Count Gloas blocks with a retained parent and fromSlot <= slot <= toSlot, and how many of them - * have a revealed payload (FULL variant exists). Used by the builder circuit breaker. + * Count Gloas blocks with fromSlot <= slot <= toSlot, and how many of them have a revealed payload + * (FULL variant exists). Used by the builder circuit breaker. */ getPayloadRevealCounts(fromSlot: Slot, toSlot: Slot): {blocksPresent: number; payloadsRevealed: number} { let blocksPresent = 0; @@ -704,9 +704,8 @@ export class ProtoArray { if (node.slot < fromSlot || node.slot > toSlot || node.payloadStatus !== PayloadStatus.PENDING) { continue; } - // Skip roots without a retained parent. This includes the anchor, which is seeded without - // payload data, and stale branch roots disconnected by pruning. - if (node.parent === undefined) { + // Genesis is always EMPTY + if (node.slot === GENESIS_SLOT) { continue; } blocksPresent++; diff --git a/packages/fork-choice/test/unit/protoArray/gloas.test.ts b/packages/fork-choice/test/unit/protoArray/gloas.test.ts index 285981795fe0..70e0c82dc8d3 100644 --- a/packages/fork-choice/test/unit/protoArray/gloas.test.ts +++ b/packages/fork-choice/test/unit/protoArray/gloas.test.ts @@ -1,6 +1,6 @@ import {beforeEach, describe, expect, it} from "vitest"; import {BitArray} from "@chainsafe/ssz"; -import {PTC_SIZE} from "@lodestar/params"; +import {GENESIS_SLOT, PTC_SIZE} from "@lodestar/params"; import {DataAvailabilityStatus, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {RootHex} from "@lodestar/types"; import {ExecutionStatus, PayloadStatus, ProtoArray, ProtoBlock, ProtoNode} from "../../../src/index.js"; @@ -227,19 +227,37 @@ describe("Gloas Fork Choice", () => { }); }); - it("does not count the anchor block seeded at initialization", () => { + it("does not count the genesis block", () => { + const currentSlot = GENESIS_SLOT + 1; + const protoArray = ProtoArray.initialize(createTestBlock(GENESIS_SLOT, genesisRoot, "0x00", "0x00"), currentSlot); + + protoArray.onBlock(createTestBlock(currentSlot, "0x02", genesisRoot, genesisRoot), currentSlot, null); + + expect(protoArray.getPayloadRevealCounts(GENESIS_SLOT, currentSlot)).toEqual({ + blocksPresent: 1, + payloadsRevealed: 0, + }); + }); + + it("counts a non-genesis anchor block seeded at initialization", () => { + const anchorRoot = "0x02"; const currentSlot = gloasForkSlot + 1; - // Gloas anchor is seeded as PENDING without payload data, as after checkpoint sync or gloas genesis - const protoArray = ProtoArray.initialize( - createTestBlock(gloasForkSlot, genesisRoot, "0x00", "0x00"), - currentSlot - ); + const protoArray = ProtoArray.initialize(createTestBlock(gloasForkSlot, anchorRoot, "0x00", "0x00"), currentSlot); - protoArray.onBlock(createTestBlock(gloasForkSlot + 1, "0x02", genesisRoot, genesisRoot), currentSlot, null); + protoArray.onExecutionPayload( + anchorRoot, + currentSlot, + "0x02ff", + 1, + 30000000, + null, + ExecutionStatus.Valid, + DataAvailabilityStatus.Available + ); expect(protoArray.getPayloadRevealCounts(gloasForkSlot, currentSlot)).toEqual({ blocksPresent: 1, - payloadsRevealed: 0, + payloadsRevealed: 1, }); }); }); From af6864fbf13f550ef7fb3a2fdbe4fd28e750bdcb Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Aug 2026 21:42:01 +0100 Subject: [PATCH 16/18] nit --- packages/fork-choice/src/protoArray/protoArray.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index 012f0fad5569..990d2e61a2da 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -704,7 +704,7 @@ export class ProtoArray { if (node.slot < fromSlot || node.slot > toSlot || node.payloadStatus !== PayloadStatus.PENDING) { continue; } - // Genesis is always EMPTY + // Genesis block is always EMPTY if (node.slot === GENESIS_SLOT) { continue; } From affd1519f3eb10e6d7c05b2d29cf70f29f8f6568 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 13 Aug 2026 11:01:05 +0100 Subject: [PATCH 17/18] fix: count canonical FULL and EMPTY blocks in circuit breaker --- .../src/chain/builderCircuitBreaker.ts | 16 +- packages/beacon-node/src/chain/options.ts | 2 +- .../beacon-node/src/metrics/metrics/beacon.ts | 10 +- .../test/mocks/mockedBeaconChain.ts | 2 +- .../unit/chain/builderCircuitBreaker.test.ts | 65 ++++-- .../src/options/beaconNodeOptions/builder.ts | 4 +- .../fork-choice/src/forkChoice/forkChoice.ts | 4 +- .../fork-choice/src/forkChoice/interface.ts | 7 +- .../fork-choice/src/protoArray/protoArray.ts | 45 ++-- .../test/unit/protoArray/gloas.test.ts | 202 +++++++----------- 10 files changed, 164 insertions(+), 193 deletions(-) diff --git a/packages/beacon-node/src/chain/builderCircuitBreaker.ts b/packages/beacon-node/src/chain/builderCircuitBreaker.ts index b632fc182088..c59ca6f0ee6c 100644 --- a/packages/beacon-node/src/chain/builderCircuitBreaker.ts +++ b/packages/beacon-node/src/chain/builderCircuitBreaker.ts @@ -21,9 +21,9 @@ const MIN_BLOCKS_TO_DEACTIVATE = 4; /** * Post-gloas circuit breaker for builder bids. The beacon block is produced by the proposer * regardless of bid source, so missed blocks are not a useful builder health signal. Instead - * count blocks whose payload was never revealed. Activate when the non-reveal rate exceeds the - * fault budget, and resume selecting builder bids only when the observed blocks are within budget - * and meet the minimum recovery sample size. + * count canonical blocks selected as EMPTY. Activate when the proportion of EMPTY blocks exceeds + * the fault budget, and resume selecting builder bids only when the observed blocks are within + * budget and meet the minimum recovery sample size. */ export class BuilderCircuitBreaker { readonly faultInspectionWindow: number; @@ -39,6 +39,7 @@ export class BuilderCircuitBreaker { const {faultInspectionWindow, allowedFaults} = getFaultInspectionParams(opts); this.faultInspectionWindow = faultInspectionWindow; this.allowedFaults = allowedFaults; + this.modules.logger.info("Builder circuit breaker initialized", {faultInspectionWindow, allowedFaults}); } /** Whether builder bids must be ignored for a block produced at clockSlot */ @@ -53,12 +54,13 @@ export class BuilderCircuitBreaker { } this.lastUpdatedSlot = clockSlot; - // Exclude clockSlot itself, its payload reveal may still be in flight - const {blocksPresent, payloadsRevealed} = this.modules.forkChoice.getPayloadRevealCounts( + // Exclude clockSlot itself, its payload status may still be unresolved + const {full, empty} = this.modules.forkChoice.getCanonicalPayloadCounts( Math.max(clockSlot - this.faultInspectionWindow, 0), clockSlot - 1 ); - const faults = blocksPresent - payloadsRevealed; + const blocksPresent = full + empty; + const faults = empty; const wasActive = this.active; // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates @@ -73,7 +75,7 @@ export class BuilderCircuitBreaker { this.modules.metrics?.builderCircuitBreaker.active.set(this.active ? 1 : 0); this.modules.metrics?.builderCircuitBreaker.faults.set(faults); this.modules.metrics?.builderCircuitBreaker.blocksPresent.set(blocksPresent); - this.modules.metrics?.builderCircuitBreaker.payloadsRevealed.set(payloadsRevealed); + this.modules.metrics?.builderCircuitBreaker.payloadsFull.set(full); const logCtx = { clockSlot, diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index 8a1267126601..b836b48d76cb 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -51,7 +51,7 @@ export type IChainOptions = BlockProcessOpts & nativeStateView?: boolean; /** Builder circuit breaker fault inspection window in slots */ faultInspectionWindow?: number; - /** Unrevealed payloads allowed per `faultInspectionWindow` observed blocks */ + /** Canonical EMPTY blocks allowed per `faultInspectionWindow` observed blocks */ allowedFaults?: number; }; diff --git a/packages/beacon-node/src/metrics/metrics/beacon.ts b/packages/beacon-node/src/metrics/metrics/beacon.ts index a2e37616d386..5f42f822ee6d 100644 --- a/packages/beacon-node/src/metrics/metrics/beacon.ts +++ b/packages/beacon-node/src/metrics/metrics/beacon.ts @@ -145,15 +145,15 @@ export function createBeaconMetrics(register: RegistryMetricCreator) { }), faults: register.gauge({ name: "beacon_builder_circuit_breaker_faults", - help: "Count of blocks with unrevealed payloads in the fault inspection window", + help: "Count of canonical blocks resolved EMPTY in the fault inspection window", }), blocksPresent: register.gauge({ name: "beacon_builder_circuit_breaker_blocks_present", - help: "Count of blocks present in the fault inspection window", + help: "Count of canonical blocks with resolved payload status in the fault inspection window", }), - payloadsRevealed: register.gauge({ - name: "beacon_builder_circuit_breaker_payloads_revealed", - help: "Count of blocks with revealed payloads in the fault inspection window", + payloadsFull: register.gauge({ + name: "beacon_builder_circuit_breaker_payloads_full", + help: "Count of canonical blocks resolved FULL in the fault inspection window", }), }, diff --git a/packages/beacon-node/test/mocks/mockedBeaconChain.ts b/packages/beacon-node/test/mocks/mockedBeaconChain.ts index 0c1f9b77bbfe..857907097344 100644 --- a/packages/beacon-node/test/mocks/mockedBeaconChain.ts +++ b/packages/beacon-node/test/mocks/mockedBeaconChain.ts @@ -78,7 +78,7 @@ vi.mock("@lodestar/fork-choice", async (importActual) => { getBlockSummariesAtSlot: vi.fn(), notifyPtcMessages: vi.fn(), shouldBuildOnFull: vi.fn(), - getPayloadRevealCounts: vi.fn(), + getCanonicalPayloadCounts: vi.fn(), }; }); diff --git a/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts b/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts index 3f7c53b81fae..94d9bc488bf6 100644 --- a/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts +++ b/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts @@ -4,31 +4,32 @@ import {testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {BuilderCircuitBreaker} from "../../../src/chain/builderCircuitBreaker.js"; import {getFaultInspectionParams} from "../../../src/execution/builder/http.js"; +import {getMockedLogger} from "../../mocks/loggerMock.js"; describe("BuilderCircuitBreaker", () => { const faultInspectionWindow = 32; const allowedFaults = 8; const logger = testLogger("builderCircuitBreaker"); - function setup(stats: {blocksPresent: number; payloadsRevealed: number}) { - const getPayloadRevealCounts = vi.fn().mockReturnValue(stats); - const forkChoice = {getPayloadRevealCounts} as unknown as IForkChoice; + function setup(counts: {full: number; empty: number}) { + const getCanonicalPayloadCounts = vi.fn().mockReturnValue(counts); + const forkChoice = {getCanonicalPayloadCounts} as unknown as IForkChoice; const breaker = new BuilderCircuitBreaker( {faultInspectionWindow, allowedFaults}, {forkChoice, logger, metrics: null} ); - return {breaker, getPayloadRevealCounts}; + return {breaker, getCanonicalPayloadCounts}; } - const testCases: [string, {blocksPresent: number; payloadsRevealed: number}, boolean][] = [ - ["empty window keeps initial state", {blocksPresent: 0, payloadsRevealed: 0}, false], - ["full window, no faults", {blocksPresent: 32, payloadsRevealed: 32}, false], - ["full window, faults at budget", {blocksPresent: 32, payloadsRevealed: 24}, false], - ["full window, faults above budget", {blocksPresent: 32, payloadsRevealed: 23}, true], - ["sparse window, faults within scaled budget", {blocksPresent: 8, payloadsRevealed: 6}, false], - ["sparse window, faults above scaled budget", {blocksPresent: 8, payloadsRevealed: 5}, true], - ["single unrevealed payload", {blocksPresent: 1, payloadsRevealed: 0}, true], - ["sparse window, all payloads unrevealed", {blocksPresent: 4, payloadsRevealed: 0}, true], + const testCases: [string, {full: number; empty: number}, boolean][] = [ + ["empty window keeps initial state", {full: 0, empty: 0}, false], + ["full window, no faults", {full: 32, empty: 0}, false], + ["full window, faults at budget", {full: 24, empty: 8}, false], + ["full window, faults above budget", {full: 23, empty: 9}, true], + ["sparse window, faults within scaled budget", {full: 6, empty: 2}, false], + ["sparse window, faults above scaled budget", {full: 5, empty: 3}, true], + ["single EMPTY block", {full: 0, empty: 1}, true], + ["sparse window, all blocks EMPTY", {full: 0, empty: 4}, true], ]; for (const [name, stats, expected] of testCases) { @@ -38,36 +39,54 @@ describe("BuilderCircuitBreaker", () => { }); } + it("logs the resolved configuration on initialization", () => { + const logger = getMockedLogger(); + + new BuilderCircuitBreaker( + {faultInspectionWindow, allowedFaults}, + { + forkChoice: {getCanonicalPayloadCounts: vi.fn()} as unknown as IForkChoice, + logger, + metrics: null, + } + ); + + expect(logger.info).toHaveBeenCalledWith("Builder circuit breaker initialized", { + faultInspectionWindow, + allowedFaults, + }); + }); + it("inspects the window excluding the current slot", () => { - const {breaker, getPayloadRevealCounts} = setup({blocksPresent: 32, payloadsRevealed: 32}); + const {breaker, getCanonicalPayloadCounts} = setup({full: 32, empty: 0}); breaker.isActive(100); - expect(getPayloadRevealCounts).toHaveBeenCalledWith(100 - faultInspectionWindow, 99); + expect(getCanonicalPayloadCounts).toHaveBeenCalledWith(100 - faultInspectionWindow, 99); }); it("requires a minimum sample to deactivate", () => { - const {breaker, getPayloadRevealCounts} = setup({blocksPresent: 1, payloadsRevealed: 0}); + const {breaker, getCanonicalPayloadCounts} = setup({full: 0, empty: 1}); expect(breaker.isActive(100)).toBe(true); - getPayloadRevealCounts.mockReturnValue({blocksPresent: 0, payloadsRevealed: 0}); + getCanonicalPayloadCounts.mockReturnValue({full: 0, empty: 0}); expect(breaker.isActive(101)).toBe(true); - getPayloadRevealCounts.mockReturnValue({blocksPresent: 3, payloadsRevealed: 3}); + getCanonicalPayloadCounts.mockReturnValue({full: 3, empty: 0}); expect(breaker.isActive(102)).toBe(true); - getPayloadRevealCounts.mockReturnValue({blocksPresent: 4, payloadsRevealed: 3}); + getCanonicalPayloadCounts.mockReturnValue({full: 3, empty: 1}); expect(breaker.isActive(103)).toBe(false); }); it("only updates once per slot", () => { - const {breaker, getPayloadRevealCounts} = setup({blocksPresent: 32, payloadsRevealed: 32}); + const {breaker, getCanonicalPayloadCounts} = setup({full: 32, empty: 0}); expect(breaker.isActive(100)).toBe(false); - getPayloadRevealCounts.mockReturnValue({blocksPresent: 32, payloadsRevealed: 0}); + getCanonicalPayloadCounts.mockReturnValue({full: 0, empty: 32}); expect(breaker.isActive(100)).toBe(false); - expect(getPayloadRevealCounts).toHaveBeenCalledTimes(1); + expect(getCanonicalPayloadCounts).toHaveBeenCalledTimes(1); expect(breaker.isActive(101)).toBe(true); - expect(getPayloadRevealCounts).toHaveBeenCalledTimes(2); + expect(getCanonicalPayloadCounts).toHaveBeenCalledTimes(2); }); describe("getFaultInspectionParams", () => { diff --git a/packages/cli/src/options/beaconNodeOptions/builder.ts b/packages/cli/src/options/beaconNodeOptions/builder.ts index efc5bd4fbcf2..07ab377d67e8 100644 --- a/packages/cli/src/options/beaconNodeOptions/builder.ts +++ b/packages/cli/src/options/beaconNodeOptions/builder.ts @@ -55,14 +55,14 @@ export const options: CliCommandOptions = { "builder.faultInspectionWindow": { type: "number", description: - "Window to inspect missed slots (pre-gloas) or unrevealed payloads (post-gloas) for enabling/disabling builder circuit breaker", + "Window to inspect missed slots (pre-gloas) or canonical blocks selected as FULL/EMPTY (post-gloas) for enabling/disabling builder circuit breaker", group: "builder", }, "builder.allowedFaults": { type: "number", description: - "Number of missed slots allowed within `faultInspectionWindow` before ignoring the external builder (pre-gloas). Post-gloas, sets the tolerated rate of unrevealed payloads, defined as `allowedFaults` out of `faultInspectionWindow` and applied to blocks observed in the window", + "Number of missed slots allowed within `faultInspectionWindow` before ignoring the external builder (pre-gloas). Post-gloas, sets the tolerated rate of canonical blocks resolved EMPTY, defined as `allowedFaults` out of `faultInspectionWindow` and applied to resolved blocks in the window", group: "builder", }, }; diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index 3732f848111d..d3579f7cb75e 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -614,8 +614,8 @@ export class ForkChoice implements IForkChoice { return this.protoArray.nodes.filter((node) => node.slot > windowStart).length; } - getPayloadRevealCounts(fromSlot: Slot, toSlot: Slot): {blocksPresent: number; payloadsRevealed: number} { - return this.protoArray.getPayloadRevealCounts(fromSlot, toSlot); + getCanonicalPayloadCounts(fromSlot: Slot, toSlot: Slot): {full: number; empty: number} { + return this.protoArray.getCanonicalPayloadCounts(fromSlot, toSlot, this.head.blockRoot, this.head.payloadStatus); } /** Very expensive function, iterates the entire ProtoArray. Called only in debug API */ diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index 2c78375cc79d..01b632463555 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -246,11 +246,8 @@ export interface IForkChoice { hasPayloadUnsafe(blockRoot: Root): boolean; hasPayloadHexUnsafe(blockRoot: RootHex): boolean; getSlotsPresent(windowStart: number): number; - /** - * Count Gloas blocks with fromSlot <= slot <= toSlot, and how many of them have a revealed payload - * (FULL variant exists). Used by the builder circuit breaker. - */ - getPayloadRevealCounts(fromSlot: Slot, toSlot: Slot): {blocksPresent: number; payloadsRevealed: number}; + /** Count canonical Gloas blocks selected as FULL or EMPTY in the inclusive slot range. */ + getCanonicalPayloadCounts(fromSlot: Slot, toSlot: Slot): {full: number; empty: number}; getPTCVotes(blockRootHex: RootHex): (boolean | null)[] | null; /** Raw PTC vote tallies for the debug fork choice endpoint; `null` for pre-Gloas roots. */ getPTCVoteCounts(blockRootHex: RootHex): { diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index 990d2e61a2da..41b3b73076b7 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -690,30 +690,35 @@ export class ProtoArray { this.maybeUpdateBestChildAndDescendant(pendingIndex, fullIndex, currentSlot, proposerBoostRoot); } - /** - * Count Gloas blocks with fromSlot <= slot <= toSlot, and how many of them have a revealed payload - * (FULL variant exists). Used by the builder circuit breaker. - */ - getPayloadRevealCounts(fromSlot: Slot, toSlot: Slot): {blocksPresent: number; payloadsRevealed: number} { - let blocksPresent = 0; - let payloadsRevealed = 0; - // Full scan, nodes are in import order not slot order (an old block can be imported after newer - // ones during sync or reorg resolution), so we cannot stop early on an out-of-window slot - for (const node of this.nodes) { - // Count each gloas block once via its PENDING variant, pre-gloas nodes are FULL only - if (node.slot < fromSlot || node.slot > toSlot || node.payloadStatus !== PayloadStatus.PENDING) { - continue; - } - // Genesis block is always EMPTY - if (node.slot === GENESIS_SLOT) { + /** Count Gloas blocks selected as FULL or EMPTY by the supplied head chain in the inclusive slot range. */ + getCanonicalPayloadCounts( + fromSlot: Slot, + toSlot: Slot, + headRoot: RootHex, + headPayloadStatus: PayloadStatus + ): {full: number; empty: number} { + let full = 0; + let empty = 0; + + for (const node of this.getAllAncestorNodes(headRoot, headPayloadStatus)) { + if ( + node.slot === GENESIS_SLOT || + node.slot < fromSlot || + node.slot > toSlot || + !isGloasBlock(node) || + node.payloadStatus === PayloadStatus.PENDING + ) { continue; } - blocksPresent++; - if (this.hasPayload(node.blockRoot)) { - payloadsRevealed++; + + if (node.payloadStatus === PayloadStatus.FULL) { + full++; + } else { + empty++; } } - return {blocksPresent, payloadsRevealed}; + + return {full, empty}; } /** diff --git a/packages/fork-choice/test/unit/protoArray/gloas.test.ts b/packages/fork-choice/test/unit/protoArray/gloas.test.ts index 70e0c82dc8d3..a16391866b84 100644 --- a/packages/fork-choice/test/unit/protoArray/gloas.test.ts +++ b/packages/fork-choice/test/unit/protoArray/gloas.test.ts @@ -1,6 +1,6 @@ import {beforeEach, describe, expect, it} from "vitest"; import {BitArray} from "@chainsafe/ssz"; -import {GENESIS_SLOT, PTC_SIZE} from "@lodestar/params"; +import {PTC_SIZE} from "@lodestar/params"; import {DataAvailabilityStatus, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {RootHex} from "@lodestar/types"; import {ExecutionStatus, PayloadStatus, ProtoArray, ProtoBlock, ProtoNode} from "../../../src/index.js"; @@ -92,66 +92,16 @@ describe("Gloas Fork Choice", () => { }); }); - describe("getPayloadRevealCounts", () => { - it("counts blocks and revealed payloads within slot range", () => { + describe("getCanonicalPayloadCounts", () => { + it("excludes competing branches and keeps EMPTY after a late FULL arrives", () => { const currentSlot = gloasForkSlot + 2; const protoArray = ProtoArray.initialize( createTestBlock(gloasForkSlot - 1, genesisRoot, "0x00"), gloasForkSlot - 1 ); - const gloasBlocks = [ - createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot), - createTestBlock(gloasForkSlot + 1, "0x03", genesisRoot, genesisRoot), - createTestBlock(gloasForkSlot + 2, "0x04", genesisRoot, genesisRoot), - ]; - for (const block of gloasBlocks) { - protoArray.onBlock(block, currentSlot, null); - } - // Reveal payloads for the first two blocks only - for (const blockRoot of ["0x02", "0x03"]) { - protoArray.onExecutionPayload( - blockRoot, - currentSlot, - `${blockRoot}ff`, - 1, - 30000000, - null, - ExecutionStatus.Valid, - DataAvailabilityStatus.Available - ); - } - - // Pre-gloas anchor block is not counted - expect(protoArray.getPayloadRevealCounts(0, currentSlot)).toEqual({blocksPresent: 3, payloadsRevealed: 2}); - // Slot range bounds are inclusive - expect(protoArray.getPayloadRevealCounts(gloasForkSlot + 1, gloasForkSlot + 1)).toEqual({ - blocksPresent: 1, - payloadsRevealed: 1, - }); - expect(protoArray.getPayloadRevealCounts(gloasForkSlot + 2, currentSlot + 10)).toEqual({ - blocksPresent: 1, - payloadsRevealed: 0, - }); - expect(protoArray.getPayloadRevealCounts(currentSlot + 1, currentSlot + 10)).toEqual({ - blocksPresent: 0, - payloadsRevealed: 0, - }); - }); - - // Counting all branches is intentional, it keeps the count independent of which branch is - // head at evaluation time and errs toward local building when forks are frequent - it("counts competing blocks at the same slot on different branches", () => { - const currentSlot = gloasForkSlot + 1; - const protoArray = ProtoArray.initialize( - createTestBlock(gloasForkSlot - 1, genesisRoot, "0x00"), - gloasForkSlot - 1 - ); - - // Two blocks at the same slot on competing branches, as observed on devnets - for (const blockRoot of ["0x02", "0x03"]) { - protoArray.onBlock(createTestBlock(gloasForkSlot, blockRoot, genesisRoot, genesisRoot), currentSlot, null); - } + const parent = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(parent, currentSlot, null); protoArray.onExecutionPayload( "0x02", currentSlot, @@ -163,89 +113,59 @@ describe("Gloas Fork Choice", () => { DataAvailabilityStatus.Available ); - // Both branches are assessed, not just the one that ends up on the head branch - expect(protoArray.getPayloadRevealCounts(gloasForkSlot, currentSlot)).toEqual({ - blocksPresent: 2, - payloadsRevealed: 1, - }); - }); - - it("keeps counting past FULL variants appended below the window", () => { - const currentSlot = gloasForkSlot + 3; - const protoArray = ProtoArray.initialize( - createTestBlock(gloasForkSlot - 1, genesisRoot, "0x00"), - gloasForkSlot - 1 + // The canonical child extends FULL. Two competing children are excluded regardless of whether + // their own payload was revealed. + protoArray.onBlock(createTestBlock(gloasForkSlot + 1, "0x03", "0x02", "0x02ff"), currentSlot, null); + protoArray.onBlock(createTestBlock(gloasForkSlot + 1, "0x04", "0x02", "0x02"), currentSlot, null); + protoArray.onBlock(createTestBlock(gloasForkSlot + 1, "0x06", "0x02", "0x02ff"), currentSlot, null); + protoArray.onExecutionPayload( + "0x04", + currentSlot, + "0x04ff", + 1, + 30000000, + null, + ExecutionStatus.Valid, + DataAvailabilityStatus.Available ); - // Block below the window plus two within it - for (const [slot, blockRoot] of [ - [gloasForkSlot, "0x02"], - [gloasForkSlot + 2, "0x03"], - [gloasForkSlot + 3, "0x04"], - ] as const) { - protoArray.onBlock(createTestBlock(slot, blockRoot, genesisRoot, genesisRoot), currentSlot, null); - } + // A later canonical block commits to EMPTY for 0x03. + protoArray.onBlock(createTestBlock(gloasForkSlot + 2, "0x05", "0x03", "0x03"), currentSlot, null); - // Reveal the below-window payload last so its FULL node is appended after the in-window - // nodes, the scan must skip it instead of counting or stopping on it - for (const blockRoot of ["0x04", "0x02"]) { - protoArray.onExecutionPayload( - blockRoot, - currentSlot, - `${blockRoot}ff`, - 1, - 30000000, - null, - ExecutionStatus.Valid, - DataAvailabilityStatus.Available - ); - } + // The payload for 0x03 arrives after its EMPTY variant was extended. + protoArray.onExecutionPayload( + "0x03", + currentSlot, + "0x03ff", + 1, + 30000000, + null, + ExecutionStatus.Valid, + DataAvailabilityStatus.Available + ); - expect(protoArray.getPayloadRevealCounts(gloasForkSlot + 2, currentSlot)).toEqual({ - blocksPresent: 2, - payloadsRevealed: 1, + // Only the chain selected by 0x05 is assessed. 0x02 resolved FULL and 0x03 resolved EMPTY. + expect(protoArray.getCanonicalPayloadCounts(gloasForkSlot, currentSlot, "0x05", PayloadStatus.PENDING)).toEqual({ + full: 1, + empty: 1, }); + + // Slot range bounds are inclusive. + expect( + protoArray.getCanonicalPayloadCounts(gloasForkSlot + 1, gloasForkSlot + 1, "0x05", PayloadStatus.PENDING) + ).toEqual({full: 0, empty: 1}); }); - it("counts in-window blocks even when an older block is imported afterwards", () => { - const currentSlot = gloasForkSlot + 3; + it("uses the supplied head branch", () => { + const currentSlot = gloasForkSlot + 1; const protoArray = ProtoArray.initialize( createTestBlock(gloasForkSlot - 1, genesisRoot, "0x00"), gloasForkSlot - 1 ); - // Two in-window blocks - protoArray.onBlock(createTestBlock(gloasForkSlot + 2, "0x03", genesisRoot, genesisRoot), currentSlot, null); - protoArray.onBlock(createTestBlock(gloasForkSlot + 3, "0x04", genesisRoot, genesisRoot), currentSlot, null); - // Old block below the window imported last, so its PENDING node lands at the end of the array. - // The scan must not stop on it and miss the in-window nodes inserted earlier (nodes are not slot ordered) protoArray.onBlock(createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot), currentSlot, null); - - expect(protoArray.getPayloadRevealCounts(gloasForkSlot + 2, currentSlot)).toEqual({ - blocksPresent: 2, - payloadsRevealed: 0, - }); - }); - - it("does not count the genesis block", () => { - const currentSlot = GENESIS_SLOT + 1; - const protoArray = ProtoArray.initialize(createTestBlock(GENESIS_SLOT, genesisRoot, "0x00", "0x00"), currentSlot); - - protoArray.onBlock(createTestBlock(currentSlot, "0x02", genesisRoot, genesisRoot), currentSlot, null); - - expect(protoArray.getPayloadRevealCounts(GENESIS_SLOT, currentSlot)).toEqual({ - blocksPresent: 1, - payloadsRevealed: 0, - }); - }); - - it("counts a non-genesis anchor block seeded at initialization", () => { - const anchorRoot = "0x02"; - const currentSlot = gloasForkSlot + 1; - const protoArray = ProtoArray.initialize(createTestBlock(gloasForkSlot, anchorRoot, "0x00", "0x00"), currentSlot); - protoArray.onExecutionPayload( - anchorRoot, + "0x02", currentSlot, "0x02ff", 1, @@ -254,12 +174,40 @@ describe("Gloas Fork Choice", () => { ExecutionStatus.Valid, DataAvailabilityStatus.Available ); + protoArray.onBlock(createTestBlock(gloasForkSlot + 1, "0x03", "0x02", "0x02"), currentSlot, null); + protoArray.onBlock(createTestBlock(gloasForkSlot + 1, "0x04", "0x02", "0x02ff"), currentSlot, null); + + protoArray.onExecutionPayload( + "0x04", + currentSlot, + "0x04ff", + 1, + 30000000, + null, + ExecutionStatus.Valid, + DataAvailabilityStatus.Available + ); - expect(protoArray.getPayloadRevealCounts(gloasForkSlot, currentSlot)).toEqual({ - blocksPresent: 1, - payloadsRevealed: 1, + expect(protoArray.getCanonicalPayloadCounts(gloasForkSlot, currentSlot, "0x04", PayloadStatus.FULL)).toEqual({ + full: 2, + empty: 0, }); }); + + it("does not assess a PENDING head before its payload status is selected", () => { + const protoArray = ProtoArray.initialize( + createTestBlock(gloasForkSlot - 1, genesisRoot, "0x00"), + gloasForkSlot - 1 + ); + protoArray.onBlock(createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot), gloasForkSlot, null); + + expect(protoArray.getCanonicalPayloadCounts(gloasForkSlot, gloasForkSlot, "0x02", PayloadStatus.PENDING)).toEqual( + { + full: 0, + empty: 0, + } + ); + }); }); describe("Pre-Gloas (Fulu) behavior", () => { From a7f9aed64506a2f2694e34e26c8e215c433b26d2 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 13 Aug 2026 11:02:29 +0100 Subject: [PATCH 18/18] Revert "fix: count canonical FULL and EMPTY blocks in circuit breaker" This reverts commit affd1519f3eb10e6d7c05b2d29cf70f29f8f6568. --- .../src/chain/builderCircuitBreaker.ts | 16 +- packages/beacon-node/src/chain/options.ts | 2 +- .../beacon-node/src/metrics/metrics/beacon.ts | 10 +- .../test/mocks/mockedBeaconChain.ts | 2 +- .../unit/chain/builderCircuitBreaker.test.ts | 65 ++---- .../src/options/beaconNodeOptions/builder.ts | 4 +- .../fork-choice/src/forkChoice/forkChoice.ts | 4 +- .../fork-choice/src/forkChoice/interface.ts | 7 +- .../fork-choice/src/protoArray/protoArray.ts | 45 ++-- .../test/unit/protoArray/gloas.test.ts | 202 +++++++++++------- 10 files changed, 193 insertions(+), 164 deletions(-) diff --git a/packages/beacon-node/src/chain/builderCircuitBreaker.ts b/packages/beacon-node/src/chain/builderCircuitBreaker.ts index c59ca6f0ee6c..b632fc182088 100644 --- a/packages/beacon-node/src/chain/builderCircuitBreaker.ts +++ b/packages/beacon-node/src/chain/builderCircuitBreaker.ts @@ -21,9 +21,9 @@ const MIN_BLOCKS_TO_DEACTIVATE = 4; /** * Post-gloas circuit breaker for builder bids. The beacon block is produced by the proposer * regardless of bid source, so missed blocks are not a useful builder health signal. Instead - * count canonical blocks selected as EMPTY. Activate when the proportion of EMPTY blocks exceeds - * the fault budget, and resume selecting builder bids only when the observed blocks are within - * budget and meet the minimum recovery sample size. + * count blocks whose payload was never revealed. Activate when the non-reveal rate exceeds the + * fault budget, and resume selecting builder bids only when the observed blocks are within budget + * and meet the minimum recovery sample size. */ export class BuilderCircuitBreaker { readonly faultInspectionWindow: number; @@ -39,7 +39,6 @@ export class BuilderCircuitBreaker { const {faultInspectionWindow, allowedFaults} = getFaultInspectionParams(opts); this.faultInspectionWindow = faultInspectionWindow; this.allowedFaults = allowedFaults; - this.modules.logger.info("Builder circuit breaker initialized", {faultInspectionWindow, allowedFaults}); } /** Whether builder bids must be ignored for a block produced at clockSlot */ @@ -54,13 +53,12 @@ export class BuilderCircuitBreaker { } this.lastUpdatedSlot = clockSlot; - // Exclude clockSlot itself, its payload status may still be unresolved - const {full, empty} = this.modules.forkChoice.getCanonicalPayloadCounts( + // Exclude clockSlot itself, its payload reveal may still be in flight + const {blocksPresent, payloadsRevealed} = this.modules.forkChoice.getPayloadRevealCounts( Math.max(clockSlot - this.faultInspectionWindow, 0), clockSlot - 1 ); - const blocksPresent = full + empty; - const faults = empty; + const faults = blocksPresent - payloadsRevealed; const wasActive = this.active; // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates @@ -75,7 +73,7 @@ export class BuilderCircuitBreaker { this.modules.metrics?.builderCircuitBreaker.active.set(this.active ? 1 : 0); this.modules.metrics?.builderCircuitBreaker.faults.set(faults); this.modules.metrics?.builderCircuitBreaker.blocksPresent.set(blocksPresent); - this.modules.metrics?.builderCircuitBreaker.payloadsFull.set(full); + this.modules.metrics?.builderCircuitBreaker.payloadsRevealed.set(payloadsRevealed); const logCtx = { clockSlot, diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index b836b48d76cb..8a1267126601 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -51,7 +51,7 @@ export type IChainOptions = BlockProcessOpts & nativeStateView?: boolean; /** Builder circuit breaker fault inspection window in slots */ faultInspectionWindow?: number; - /** Canonical EMPTY blocks allowed per `faultInspectionWindow` observed blocks */ + /** Unrevealed payloads allowed per `faultInspectionWindow` observed blocks */ allowedFaults?: number; }; diff --git a/packages/beacon-node/src/metrics/metrics/beacon.ts b/packages/beacon-node/src/metrics/metrics/beacon.ts index 5f42f822ee6d..a2e37616d386 100644 --- a/packages/beacon-node/src/metrics/metrics/beacon.ts +++ b/packages/beacon-node/src/metrics/metrics/beacon.ts @@ -145,15 +145,15 @@ export function createBeaconMetrics(register: RegistryMetricCreator) { }), faults: register.gauge({ name: "beacon_builder_circuit_breaker_faults", - help: "Count of canonical blocks resolved EMPTY in the fault inspection window", + help: "Count of blocks with unrevealed payloads in the fault inspection window", }), blocksPresent: register.gauge({ name: "beacon_builder_circuit_breaker_blocks_present", - help: "Count of canonical blocks with resolved payload status in the fault inspection window", + help: "Count of blocks present in the fault inspection window", }), - payloadsFull: register.gauge({ - name: "beacon_builder_circuit_breaker_payloads_full", - help: "Count of canonical blocks resolved FULL in the fault inspection window", + payloadsRevealed: register.gauge({ + name: "beacon_builder_circuit_breaker_payloads_revealed", + help: "Count of blocks with revealed payloads in the fault inspection window", }), }, diff --git a/packages/beacon-node/test/mocks/mockedBeaconChain.ts b/packages/beacon-node/test/mocks/mockedBeaconChain.ts index 857907097344..0c1f9b77bbfe 100644 --- a/packages/beacon-node/test/mocks/mockedBeaconChain.ts +++ b/packages/beacon-node/test/mocks/mockedBeaconChain.ts @@ -78,7 +78,7 @@ vi.mock("@lodestar/fork-choice", async (importActual) => { getBlockSummariesAtSlot: vi.fn(), notifyPtcMessages: vi.fn(), shouldBuildOnFull: vi.fn(), - getCanonicalPayloadCounts: vi.fn(), + getPayloadRevealCounts: vi.fn(), }; }); diff --git a/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts b/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts index 94d9bc488bf6..3f7c53b81fae 100644 --- a/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts +++ b/packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts @@ -4,32 +4,31 @@ import {testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {BuilderCircuitBreaker} from "../../../src/chain/builderCircuitBreaker.js"; import {getFaultInspectionParams} from "../../../src/execution/builder/http.js"; -import {getMockedLogger} from "../../mocks/loggerMock.js"; describe("BuilderCircuitBreaker", () => { const faultInspectionWindow = 32; const allowedFaults = 8; const logger = testLogger("builderCircuitBreaker"); - function setup(counts: {full: number; empty: number}) { - const getCanonicalPayloadCounts = vi.fn().mockReturnValue(counts); - const forkChoice = {getCanonicalPayloadCounts} as unknown as IForkChoice; + function setup(stats: {blocksPresent: number; payloadsRevealed: number}) { + const getPayloadRevealCounts = vi.fn().mockReturnValue(stats); + const forkChoice = {getPayloadRevealCounts} as unknown as IForkChoice; const breaker = new BuilderCircuitBreaker( {faultInspectionWindow, allowedFaults}, {forkChoice, logger, metrics: null} ); - return {breaker, getCanonicalPayloadCounts}; + return {breaker, getPayloadRevealCounts}; } - const testCases: [string, {full: number; empty: number}, boolean][] = [ - ["empty window keeps initial state", {full: 0, empty: 0}, false], - ["full window, no faults", {full: 32, empty: 0}, false], - ["full window, faults at budget", {full: 24, empty: 8}, false], - ["full window, faults above budget", {full: 23, empty: 9}, true], - ["sparse window, faults within scaled budget", {full: 6, empty: 2}, false], - ["sparse window, faults above scaled budget", {full: 5, empty: 3}, true], - ["single EMPTY block", {full: 0, empty: 1}, true], - ["sparse window, all blocks EMPTY", {full: 0, empty: 4}, true], + const testCases: [string, {blocksPresent: number; payloadsRevealed: number}, boolean][] = [ + ["empty window keeps initial state", {blocksPresent: 0, payloadsRevealed: 0}, false], + ["full window, no faults", {blocksPresent: 32, payloadsRevealed: 32}, false], + ["full window, faults at budget", {blocksPresent: 32, payloadsRevealed: 24}, false], + ["full window, faults above budget", {blocksPresent: 32, payloadsRevealed: 23}, true], + ["sparse window, faults within scaled budget", {blocksPresent: 8, payloadsRevealed: 6}, false], + ["sparse window, faults above scaled budget", {blocksPresent: 8, payloadsRevealed: 5}, true], + ["single unrevealed payload", {blocksPresent: 1, payloadsRevealed: 0}, true], + ["sparse window, all payloads unrevealed", {blocksPresent: 4, payloadsRevealed: 0}, true], ]; for (const [name, stats, expected] of testCases) { @@ -39,54 +38,36 @@ describe("BuilderCircuitBreaker", () => { }); } - it("logs the resolved configuration on initialization", () => { - const logger = getMockedLogger(); - - new BuilderCircuitBreaker( - {faultInspectionWindow, allowedFaults}, - { - forkChoice: {getCanonicalPayloadCounts: vi.fn()} as unknown as IForkChoice, - logger, - metrics: null, - } - ); - - expect(logger.info).toHaveBeenCalledWith("Builder circuit breaker initialized", { - faultInspectionWindow, - allowedFaults, - }); - }); - it("inspects the window excluding the current slot", () => { - const {breaker, getCanonicalPayloadCounts} = setup({full: 32, empty: 0}); + const {breaker, getPayloadRevealCounts} = setup({blocksPresent: 32, payloadsRevealed: 32}); breaker.isActive(100); - expect(getCanonicalPayloadCounts).toHaveBeenCalledWith(100 - faultInspectionWindow, 99); + expect(getPayloadRevealCounts).toHaveBeenCalledWith(100 - faultInspectionWindow, 99); }); it("requires a minimum sample to deactivate", () => { - const {breaker, getCanonicalPayloadCounts} = setup({full: 0, empty: 1}); + const {breaker, getPayloadRevealCounts} = setup({blocksPresent: 1, payloadsRevealed: 0}); expect(breaker.isActive(100)).toBe(true); - getCanonicalPayloadCounts.mockReturnValue({full: 0, empty: 0}); + getPayloadRevealCounts.mockReturnValue({blocksPresent: 0, payloadsRevealed: 0}); expect(breaker.isActive(101)).toBe(true); - getCanonicalPayloadCounts.mockReturnValue({full: 3, empty: 0}); + getPayloadRevealCounts.mockReturnValue({blocksPresent: 3, payloadsRevealed: 3}); expect(breaker.isActive(102)).toBe(true); - getCanonicalPayloadCounts.mockReturnValue({full: 3, empty: 1}); + getPayloadRevealCounts.mockReturnValue({blocksPresent: 4, payloadsRevealed: 3}); expect(breaker.isActive(103)).toBe(false); }); it("only updates once per slot", () => { - const {breaker, getCanonicalPayloadCounts} = setup({full: 32, empty: 0}); + const {breaker, getPayloadRevealCounts} = setup({blocksPresent: 32, payloadsRevealed: 32}); expect(breaker.isActive(100)).toBe(false); - getCanonicalPayloadCounts.mockReturnValue({full: 0, empty: 32}); + getPayloadRevealCounts.mockReturnValue({blocksPresent: 32, payloadsRevealed: 0}); expect(breaker.isActive(100)).toBe(false); - expect(getCanonicalPayloadCounts).toHaveBeenCalledTimes(1); + expect(getPayloadRevealCounts).toHaveBeenCalledTimes(1); expect(breaker.isActive(101)).toBe(true); - expect(getCanonicalPayloadCounts).toHaveBeenCalledTimes(2); + expect(getPayloadRevealCounts).toHaveBeenCalledTimes(2); }); describe("getFaultInspectionParams", () => { diff --git a/packages/cli/src/options/beaconNodeOptions/builder.ts b/packages/cli/src/options/beaconNodeOptions/builder.ts index 07ab377d67e8..efc5bd4fbcf2 100644 --- a/packages/cli/src/options/beaconNodeOptions/builder.ts +++ b/packages/cli/src/options/beaconNodeOptions/builder.ts @@ -55,14 +55,14 @@ export const options: CliCommandOptions = { "builder.faultInspectionWindow": { type: "number", description: - "Window to inspect missed slots (pre-gloas) or canonical blocks selected as FULL/EMPTY (post-gloas) for enabling/disabling builder circuit breaker", + "Window to inspect missed slots (pre-gloas) or unrevealed payloads (post-gloas) for enabling/disabling builder circuit breaker", group: "builder", }, "builder.allowedFaults": { type: "number", description: - "Number of missed slots allowed within `faultInspectionWindow` before ignoring the external builder (pre-gloas). Post-gloas, sets the tolerated rate of canonical blocks resolved EMPTY, defined as `allowedFaults` out of `faultInspectionWindow` and applied to resolved blocks in the window", + "Number of missed slots allowed within `faultInspectionWindow` before ignoring the external builder (pre-gloas). Post-gloas, sets the tolerated rate of unrevealed payloads, defined as `allowedFaults` out of `faultInspectionWindow` and applied to blocks observed in the window", group: "builder", }, }; diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index d3579f7cb75e..3732f848111d 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -614,8 +614,8 @@ export class ForkChoice implements IForkChoice { return this.protoArray.nodes.filter((node) => node.slot > windowStart).length; } - getCanonicalPayloadCounts(fromSlot: Slot, toSlot: Slot): {full: number; empty: number} { - return this.protoArray.getCanonicalPayloadCounts(fromSlot, toSlot, this.head.blockRoot, this.head.payloadStatus); + getPayloadRevealCounts(fromSlot: Slot, toSlot: Slot): {blocksPresent: number; payloadsRevealed: number} { + return this.protoArray.getPayloadRevealCounts(fromSlot, toSlot); } /** Very expensive function, iterates the entire ProtoArray. Called only in debug API */ diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index 01b632463555..2c78375cc79d 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -246,8 +246,11 @@ export interface IForkChoice { hasPayloadUnsafe(blockRoot: Root): boolean; hasPayloadHexUnsafe(blockRoot: RootHex): boolean; getSlotsPresent(windowStart: number): number; - /** Count canonical Gloas blocks selected as FULL or EMPTY in the inclusive slot range. */ - getCanonicalPayloadCounts(fromSlot: Slot, toSlot: Slot): {full: number; empty: number}; + /** + * Count Gloas blocks with fromSlot <= slot <= toSlot, and how many of them have a revealed payload + * (FULL variant exists). Used by the builder circuit breaker. + */ + getPayloadRevealCounts(fromSlot: Slot, toSlot: Slot): {blocksPresent: number; payloadsRevealed: number}; getPTCVotes(blockRootHex: RootHex): (boolean | null)[] | null; /** Raw PTC vote tallies for the debug fork choice endpoint; `null` for pre-Gloas roots. */ getPTCVoteCounts(blockRootHex: RootHex): { diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index 41b3b73076b7..990d2e61a2da 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -690,35 +690,30 @@ export class ProtoArray { this.maybeUpdateBestChildAndDescendant(pendingIndex, fullIndex, currentSlot, proposerBoostRoot); } - /** Count Gloas blocks selected as FULL or EMPTY by the supplied head chain in the inclusive slot range. */ - getCanonicalPayloadCounts( - fromSlot: Slot, - toSlot: Slot, - headRoot: RootHex, - headPayloadStatus: PayloadStatus - ): {full: number; empty: number} { - let full = 0; - let empty = 0; - - for (const node of this.getAllAncestorNodes(headRoot, headPayloadStatus)) { - if ( - node.slot === GENESIS_SLOT || - node.slot < fromSlot || - node.slot > toSlot || - !isGloasBlock(node) || - node.payloadStatus === PayloadStatus.PENDING - ) { + /** + * Count Gloas blocks with fromSlot <= slot <= toSlot, and how many of them have a revealed payload + * (FULL variant exists). Used by the builder circuit breaker. + */ + getPayloadRevealCounts(fromSlot: Slot, toSlot: Slot): {blocksPresent: number; payloadsRevealed: number} { + let blocksPresent = 0; + let payloadsRevealed = 0; + // Full scan, nodes are in import order not slot order (an old block can be imported after newer + // ones during sync or reorg resolution), so we cannot stop early on an out-of-window slot + for (const node of this.nodes) { + // Count each gloas block once via its PENDING variant, pre-gloas nodes are FULL only + if (node.slot < fromSlot || node.slot > toSlot || node.payloadStatus !== PayloadStatus.PENDING) { continue; } - - if (node.payloadStatus === PayloadStatus.FULL) { - full++; - } else { - empty++; + // Genesis block is always EMPTY + if (node.slot === GENESIS_SLOT) { + continue; + } + blocksPresent++; + if (this.hasPayload(node.blockRoot)) { + payloadsRevealed++; } } - - return {full, empty}; + return {blocksPresent, payloadsRevealed}; } /** diff --git a/packages/fork-choice/test/unit/protoArray/gloas.test.ts b/packages/fork-choice/test/unit/protoArray/gloas.test.ts index a16391866b84..70e0c82dc8d3 100644 --- a/packages/fork-choice/test/unit/protoArray/gloas.test.ts +++ b/packages/fork-choice/test/unit/protoArray/gloas.test.ts @@ -1,6 +1,6 @@ import {beforeEach, describe, expect, it} from "vitest"; import {BitArray} from "@chainsafe/ssz"; -import {PTC_SIZE} from "@lodestar/params"; +import {GENESIS_SLOT, PTC_SIZE} from "@lodestar/params"; import {DataAvailabilityStatus, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {RootHex} from "@lodestar/types"; import {ExecutionStatus, PayloadStatus, ProtoArray, ProtoBlock, ProtoNode} from "../../../src/index.js"; @@ -92,16 +92,66 @@ describe("Gloas Fork Choice", () => { }); }); - describe("getCanonicalPayloadCounts", () => { - it("excludes competing branches and keeps EMPTY after a late FULL arrives", () => { + describe("getPayloadRevealCounts", () => { + it("counts blocks and revealed payloads within slot range", () => { const currentSlot = gloasForkSlot + 2; const protoArray = ProtoArray.initialize( createTestBlock(gloasForkSlot - 1, genesisRoot, "0x00"), gloasForkSlot - 1 ); - const parent = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(parent, currentSlot, null); + const gloasBlocks = [ + createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot), + createTestBlock(gloasForkSlot + 1, "0x03", genesisRoot, genesisRoot), + createTestBlock(gloasForkSlot + 2, "0x04", genesisRoot, genesisRoot), + ]; + for (const block of gloasBlocks) { + protoArray.onBlock(block, currentSlot, null); + } + // Reveal payloads for the first two blocks only + for (const blockRoot of ["0x02", "0x03"]) { + protoArray.onExecutionPayload( + blockRoot, + currentSlot, + `${blockRoot}ff`, + 1, + 30000000, + null, + ExecutionStatus.Valid, + DataAvailabilityStatus.Available + ); + } + + // Pre-gloas anchor block is not counted + expect(protoArray.getPayloadRevealCounts(0, currentSlot)).toEqual({blocksPresent: 3, payloadsRevealed: 2}); + // Slot range bounds are inclusive + expect(protoArray.getPayloadRevealCounts(gloasForkSlot + 1, gloasForkSlot + 1)).toEqual({ + blocksPresent: 1, + payloadsRevealed: 1, + }); + expect(protoArray.getPayloadRevealCounts(gloasForkSlot + 2, currentSlot + 10)).toEqual({ + blocksPresent: 1, + payloadsRevealed: 0, + }); + expect(protoArray.getPayloadRevealCounts(currentSlot + 1, currentSlot + 10)).toEqual({ + blocksPresent: 0, + payloadsRevealed: 0, + }); + }); + + // Counting all branches is intentional, it keeps the count independent of which branch is + // head at evaluation time and errs toward local building when forks are frequent + it("counts competing blocks at the same slot on different branches", () => { + const currentSlot = gloasForkSlot + 1; + const protoArray = ProtoArray.initialize( + createTestBlock(gloasForkSlot - 1, genesisRoot, "0x00"), + gloasForkSlot - 1 + ); + + // Two blocks at the same slot on competing branches, as observed on devnets + for (const blockRoot of ["0x02", "0x03"]) { + protoArray.onBlock(createTestBlock(gloasForkSlot, blockRoot, genesisRoot, genesisRoot), currentSlot, null); + } protoArray.onExecutionPayload( "0x02", currentSlot, @@ -113,74 +163,91 @@ describe("Gloas Fork Choice", () => { DataAvailabilityStatus.Available ); - // The canonical child extends FULL. Two competing children are excluded regardless of whether - // their own payload was revealed. - protoArray.onBlock(createTestBlock(gloasForkSlot + 1, "0x03", "0x02", "0x02ff"), currentSlot, null); - protoArray.onBlock(createTestBlock(gloasForkSlot + 1, "0x04", "0x02", "0x02"), currentSlot, null); - protoArray.onBlock(createTestBlock(gloasForkSlot + 1, "0x06", "0x02", "0x02ff"), currentSlot, null); - protoArray.onExecutionPayload( - "0x04", - currentSlot, - "0x04ff", - 1, - 30000000, - null, - ExecutionStatus.Valid, - DataAvailabilityStatus.Available + // Both branches are assessed, not just the one that ends up on the head branch + expect(protoArray.getPayloadRevealCounts(gloasForkSlot, currentSlot)).toEqual({ + blocksPresent: 2, + payloadsRevealed: 1, + }); + }); + + it("keeps counting past FULL variants appended below the window", () => { + const currentSlot = gloasForkSlot + 3; + const protoArray = ProtoArray.initialize( + createTestBlock(gloasForkSlot - 1, genesisRoot, "0x00"), + gloasForkSlot - 1 ); - // A later canonical block commits to EMPTY for 0x03. - protoArray.onBlock(createTestBlock(gloasForkSlot + 2, "0x05", "0x03", "0x03"), currentSlot, null); + // Block below the window plus two within it + for (const [slot, blockRoot] of [ + [gloasForkSlot, "0x02"], + [gloasForkSlot + 2, "0x03"], + [gloasForkSlot + 3, "0x04"], + ] as const) { + protoArray.onBlock(createTestBlock(slot, blockRoot, genesisRoot, genesisRoot), currentSlot, null); + } - // The payload for 0x03 arrives after its EMPTY variant was extended. - protoArray.onExecutionPayload( - "0x03", - currentSlot, - "0x03ff", - 1, - 30000000, - null, - ExecutionStatus.Valid, - DataAvailabilityStatus.Available - ); + // Reveal the below-window payload last so its FULL node is appended after the in-window + // nodes, the scan must skip it instead of counting or stopping on it + for (const blockRoot of ["0x04", "0x02"]) { + protoArray.onExecutionPayload( + blockRoot, + currentSlot, + `${blockRoot}ff`, + 1, + 30000000, + null, + ExecutionStatus.Valid, + DataAvailabilityStatus.Available + ); + } - // Only the chain selected by 0x05 is assessed. 0x02 resolved FULL and 0x03 resolved EMPTY. - expect(protoArray.getCanonicalPayloadCounts(gloasForkSlot, currentSlot, "0x05", PayloadStatus.PENDING)).toEqual({ - full: 1, - empty: 1, + expect(protoArray.getPayloadRevealCounts(gloasForkSlot + 2, currentSlot)).toEqual({ + blocksPresent: 2, + payloadsRevealed: 1, }); - - // Slot range bounds are inclusive. - expect( - protoArray.getCanonicalPayloadCounts(gloasForkSlot + 1, gloasForkSlot + 1, "0x05", PayloadStatus.PENDING) - ).toEqual({full: 0, empty: 1}); }); - it("uses the supplied head branch", () => { - const currentSlot = gloasForkSlot + 1; + it("counts in-window blocks even when an older block is imported afterwards", () => { + const currentSlot = gloasForkSlot + 3; const protoArray = ProtoArray.initialize( createTestBlock(gloasForkSlot - 1, genesisRoot, "0x00"), gloasForkSlot - 1 ); + // Two in-window blocks + protoArray.onBlock(createTestBlock(gloasForkSlot + 2, "0x03", genesisRoot, genesisRoot), currentSlot, null); + protoArray.onBlock(createTestBlock(gloasForkSlot + 3, "0x04", genesisRoot, genesisRoot), currentSlot, null); + // Old block below the window imported last, so its PENDING node lands at the end of the array. + // The scan must not stop on it and miss the in-window nodes inserted earlier (nodes are not slot ordered) protoArray.onBlock(createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot), currentSlot, null); - protoArray.onExecutionPayload( - "0x02", - currentSlot, - "0x02ff", - 1, - 30000000, - null, - ExecutionStatus.Valid, - DataAvailabilityStatus.Available - ); - protoArray.onBlock(createTestBlock(gloasForkSlot + 1, "0x03", "0x02", "0x02"), currentSlot, null); - protoArray.onBlock(createTestBlock(gloasForkSlot + 1, "0x04", "0x02", "0x02ff"), currentSlot, null); + + expect(protoArray.getPayloadRevealCounts(gloasForkSlot + 2, currentSlot)).toEqual({ + blocksPresent: 2, + payloadsRevealed: 0, + }); + }); + + it("does not count the genesis block", () => { + const currentSlot = GENESIS_SLOT + 1; + const protoArray = ProtoArray.initialize(createTestBlock(GENESIS_SLOT, genesisRoot, "0x00", "0x00"), currentSlot); + + protoArray.onBlock(createTestBlock(currentSlot, "0x02", genesisRoot, genesisRoot), currentSlot, null); + + expect(protoArray.getPayloadRevealCounts(GENESIS_SLOT, currentSlot)).toEqual({ + blocksPresent: 1, + payloadsRevealed: 0, + }); + }); + + it("counts a non-genesis anchor block seeded at initialization", () => { + const anchorRoot = "0x02"; + const currentSlot = gloasForkSlot + 1; + const protoArray = ProtoArray.initialize(createTestBlock(gloasForkSlot, anchorRoot, "0x00", "0x00"), currentSlot); protoArray.onExecutionPayload( - "0x04", + anchorRoot, currentSlot, - "0x04ff", + "0x02ff", 1, 30000000, null, @@ -188,26 +255,11 @@ describe("Gloas Fork Choice", () => { DataAvailabilityStatus.Available ); - expect(protoArray.getCanonicalPayloadCounts(gloasForkSlot, currentSlot, "0x04", PayloadStatus.FULL)).toEqual({ - full: 2, - empty: 0, + expect(protoArray.getPayloadRevealCounts(gloasForkSlot, currentSlot)).toEqual({ + blocksPresent: 1, + payloadsRevealed: 1, }); }); - - it("does not assess a PENDING head before its payload status is selected", () => { - const protoArray = ProtoArray.initialize( - createTestBlock(gloasForkSlot - 1, genesisRoot, "0x00"), - gloasForkSlot - 1 - ); - protoArray.onBlock(createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot), gloasForkSlot, null); - - expect(protoArray.getCanonicalPayloadCounts(gloasForkSlot, gloasForkSlot, "0x02", PayloadStatus.PENDING)).toEqual( - { - full: 0, - empty: 0, - } - ); - }); }); describe("Pre-Gloas (Fulu) behavior", () => {