fix: count canonical FULL and EMPTY blocks in circuit breaker

packages/beacon-node/src/api/impl/validator/index.ts

@@ -895,7 +895,7 @@ export function getValidatorApi(
       // TODO GLOAS: add external builder api support when it is implemented
       const isBuildingOnFull = chain.forkChoice.shouldBuildOnFull(parentBlock, slot);
       const bidParentBlockHash = isBuildingOnFull ? parentBlock.executionPayloadBlockHash : parentBlock.parentBlockHash;
-      const circuitBreakerActive = chain.builderCircuitBreaker.isActive(slot);
+      const circuitBreakerActive = chain.builderCircuitBreaker.isActive(slot, parentBlock);
       // Keep a builder bid as fallback unless the circuit breaker is active
       const builderBid = circuitBreakerActive
         ? null

packages/beacon-node/src/chain/builderCircuitBreaker.ts

@@ -1,4 +1,4 @@
-import {IForkChoice} from "@lodestar/fork-choice";
+import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {Slot} from "@lodestar/types";
 import {Logger} from "@lodestar/utils";
 import {getFaultInspectionParams} from "../execution/builder/http.js";
@@ -21,9 +21,9 @@ const MIN_BLOCKS_TO_DEACTIVATE = 4;
 /**
  * Post-gloas circuit breaker for builder bids. The beacon block is produced by the proposer
  * regardless of bid source, so missed blocks are not a useful builder health signal. Instead
- * count blocks whose payload was never revealed. Activate when the non-reveal rate exceeds the
- * fault budget, and resume selecting builder bids only when the observed blocks are within budget
- * and meet the minimum recovery sample size.
+ * count canonical blocks selected as EMPTY. Activate when the proportion of EMPTY blocks exceeds
+ * the fault budget, and resume selecting builder bids only when the observed blocks are within
+ * budget and meet the minimum recovery sample size.
  */
 export class BuilderCircuitBreaker {
   readonly faultInspectionWindow: number;
@@ -39,46 +39,49 @@ export class BuilderCircuitBreaker {
     const {faultInspectionWindow, allowedFaults} = getFaultInspectionParams(opts);
     this.faultInspectionWindow = faultInspectionWindow;
     this.allowedFaults = allowedFaults;
+    this.modules.logger.info("Builder circuit breaker initialized", {faultInspectionWindow, allowedFaults});
   }
 
   /** Whether builder bids must be ignored for a block produced at clockSlot */
-  isActive(clockSlot: Slot): boolean {
-    this.update(clockSlot);
+  isActive(clockSlot: Slot, head: ProtoBlock): boolean {
+    this.update(clockSlot, head);
     return this.active;
   }
 
-  update(clockSlot: Slot): void {
+  update(clockSlot: Slot, head: ProtoBlock): void {
     if (clockSlot <= this.lastUpdatedSlot) {
       return;
     }
     this.lastUpdatedSlot = clockSlot;
 
-    // Exclude clockSlot itself, its payload reveal may still be in flight
-    const {blocksPresent, payloadsRevealed} = this.modules.forkChoice.getPayloadRevealCounts(
+    // Exclude clockSlot itself, its payload status may still be unresolved
+    const {full, empty} = this.modules.forkChoice.getCanonicalPayloadCounts(
       Math.max(clockSlot - this.faultInspectionWindow, 0),
-      clockSlot - 1
+      clockSlot - 1,
+      head
     );
-    const faults = blocksPresent - payloadsRevealed;
+    const canonicalBlocks = full + empty;
 
     const wasActive = this.active;
-    // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates
-    const exceedsFaultBudget = faults * this.faultInspectionWindow > this.allowedFaults * blocksPresent;
+    // Scale the fault budget by canonical blocks so sparse windows still trigger on high EMPTY rates
+    const exceedsFaultBudget = empty * this.faultInspectionWindow > this.allowedFaults * canonicalBlocks;
     if (exceedsFaultBudget) {
       this.active = true;
-    } else if (blocksPresent >= MIN_BLOCKS_TO_DEACTIVATE) {
+    } else if (canonicalBlocks >= MIN_BLOCKS_TO_DEACTIVATE) {
       // Require a minimum sample within the fault budget before accepting builder bids again
       this.active = false;
     }
 
     this.modules.metrics?.builderCircuitBreaker.active.set(this.active ? 1 : 0);
-    this.modules.metrics?.builderCircuitBreaker.faults.set(faults);
-    this.modules.metrics?.builderCircuitBreaker.blocksPresent.set(blocksPresent);
-    this.modules.metrics?.builderCircuitBreaker.payloadsRevealed.set(payloadsRevealed);
+    this.modules.metrics?.builderCircuitBreaker.canonicalBlocks.set(canonicalBlocks);
+    this.modules.metrics?.builderCircuitBreaker.fullBlocks.set(full);
+    this.modules.metrics?.builderCircuitBreaker.emptyBlocks.set(empty);
 
     const logCtx = {
       clockSlot,
-      blocksPresent,
-      faults,
+      canonicalBlocks,
+      full,
+      empty,
       faultInspectionWindow: this.faultInspectionWindow,
       allowedFaults: this.allowedFaults,
     };

packages/beacon-node/src/chain/options.ts

@@ -53,7 +53,7 @@ export type IChainOptions = BlockProcessOpts &
     nativeStateView?: boolean;
     /** Builder circuit breaker fault inspection window in slots */
     faultInspectionWindow?: number;
-    /** Unrevealed payloads allowed per `faultInspectionWindow` observed blocks */
+    /** Canonical EMPTY blocks allowed per `faultInspectionWindow` observed blocks */
     allowedFaults?: number;
   };
 

packages/beacon-node/src/chain/prepareNextSlot.ts

@@ -169,7 +169,7 @@ export class PrepareNextSlotScheduler {
           }
 
           if (isForkPostGloas(fork)) {
-            this.chain.builderCircuitBreaker.update(clockSlot);
+            this.chain.builderCircuitBreaker.update(clockSlot, updatedHead);
           } else {
             // Update the builder status, if enabled shoot an api call to check status
             this.chain.updateBuilderStatus(clockSlot);

packages/beacon-node/src/metrics/metrics/beacon.ts

@@ -143,17 +143,17 @@ export function createBeaconMetrics(register: RegistryMetricCreator) {
         name: "beacon_builder_circuit_breaker_active",
         help: "Whether the builder circuit breaker is active (1) causing builder bids to be ignored",
       }),
-      faults: register.gauge({
-        name: "beacon_builder_circuit_breaker_faults",
-        help: "Count of blocks with unrevealed payloads in the fault inspection window",
+      canonicalBlocks: register.gauge({
+        name: "beacon_builder_circuit_breaker_canonical_blocks",
+        help: "Count of canonical FULL and EMPTY blocks in the fault inspection window",
       }),
-      blocksPresent: register.gauge({
-        name: "beacon_builder_circuit_breaker_blocks_present",
-        help: "Count of blocks present in the fault inspection window",
+      fullBlocks: register.gauge({
+        name: "beacon_builder_circuit_breaker_full_blocks",
+        help: "Count of canonical FULL blocks in the fault inspection window",
       }),
-      payloadsRevealed: register.gauge({
-        name: "beacon_builder_circuit_breaker_payloads_revealed",
-        help: "Count of blocks with revealed payloads in the fault inspection window",
+      emptyBlocks: register.gauge({
+        name: "beacon_builder_circuit_breaker_empty_blocks",
+        help: "Count of canonical EMPTY blocks in the fault inspection window",
       }),
     },
 

packages/beacon-node/test/mocks/mockedBeaconChain.ts

@@ -78,7 +78,7 @@ vi.mock("@lodestar/fork-choice", async (importActual) => {
       getBlockSummariesAtSlot: vi.fn(),
       notifyPtcMessages: vi.fn(),
       shouldBuildOnFull: vi.fn(),
-      getPayloadRevealCounts: vi.fn(),
+      getCanonicalPayloadCounts: vi.fn(),
     };
   });
 

packages/beacon-node/test/unit/api/impl/validator/produceBlockV4.test.ts

@@ -206,7 +206,7 @@ describe("api/validator - produceBlockV4", () => {
 
     const {data: block} = await api.produceBlockV4({slot, randaoReveal, graffiti, feeRecipient, includePayload: false});
 
-    expect(modules.chain.builderCircuitBreaker.isActive).toHaveBeenCalledWith(slot);
+    expect(modules.chain.builderCircuitBreaker.isActive).toHaveBeenCalledWith(slot, parentBlock);
     expect(modules.chain.executionPayloadBidPool.getBestBid).not.toHaveBeenCalled();
     expect(modules.chain.produceBlock).toHaveBeenCalledTimes(1);
     expect(block).toEqual(engineBlock);

packages/beacon-node/test/unit/chain/builderCircuitBreaker.test.ts

@@ -1,5 +1,5 @@
 import {describe, expect, it, vi} from "vitest";
-import {IForkChoice} from "@lodestar/fork-choice";
+import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {testLogger} from "@lodestar/logger/test-utils";
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
 import {BuilderCircuitBreaker} from "../../../src/chain/builderCircuitBreaker.js";
@@ -9,65 +9,66 @@ describe("BuilderCircuitBreaker", () => {
   const faultInspectionWindow = 32;
   const allowedFaults = 8;
   const logger = testLogger("builderCircuitBreaker");
+  const head = {} as ProtoBlock;
 
-  function setup(stats: {blocksPresent: number; payloadsRevealed: number}) {
-    const getPayloadRevealCounts = vi.fn().mockReturnValue(stats);
-    const forkChoice = {getPayloadRevealCounts} as unknown as IForkChoice;
+  function setup(counts: {full: number; empty: number}) {
+    const getCanonicalPayloadCounts = vi.fn().mockReturnValue(counts);
+    const forkChoice = {getCanonicalPayloadCounts} as unknown as IForkChoice;
     const breaker = new BuilderCircuitBreaker(
       {faultInspectionWindow, allowedFaults},
       {forkChoice, logger, metrics: null}
     );
-    return {breaker, getPayloadRevealCounts};
+    return {breaker, getCanonicalPayloadCounts};
   }
 
-  const testCases: [string, {blocksPresent: number; payloadsRevealed: number}, boolean][] = [
-    ["empty window keeps initial state", {blocksPresent: 0, payloadsRevealed: 0}, false],
-    ["full window, no faults", {blocksPresent: 32, payloadsRevealed: 32}, false],
-    ["full window, faults at budget", {blocksPresent: 32, payloadsRevealed: 24}, false],
-    ["full window, faults above budget", {blocksPresent: 32, payloadsRevealed: 23}, true],
-    ["sparse window, faults within scaled budget", {blocksPresent: 8, payloadsRevealed: 6}, false],
-    ["sparse window, faults above scaled budget", {blocksPresent: 8, payloadsRevealed: 5}, true],
-    ["single unrevealed payload", {blocksPresent: 1, payloadsRevealed: 0}, true],
-    ["sparse window, all payloads unrevealed", {blocksPresent: 4, payloadsRevealed: 0}, true],
+  const testCases: [string, {full: number; empty: number}, boolean][] = [
+    ["empty window keeps initial state", {full: 0, empty: 0}, false],
+    ["full window, no faults", {full: 32, empty: 0}, false],
+    ["full window, faults at budget", {full: 24, empty: 8}, false],
+    ["full window, faults above budget", {full: 23, empty: 9}, true],
+    ["sparse window, faults within scaled budget", {full: 6, empty: 2}, false],
+    ["sparse window, faults above scaled budget", {full: 5, empty: 3}, true],
+    ["single EMPTY block", {full: 0, empty: 1}, true],
+    ["sparse window, all blocks EMPTY", {full: 0, empty: 4}, true],
   ];
 
   for (const [name, stats, expected] of testCases) {
     it(`${name} - active=${expected}`, () => {
       const {breaker} = setup(stats);
-      expect(breaker.isActive(100)).toBe(expected);
+      expect(breaker.isActive(100, head)).toBe(expected);
     });
   }
 
   it("inspects the window excluding the current slot", () => {
-    const {breaker, getPayloadRevealCounts} = setup({blocksPresent: 32, payloadsRevealed: 32});
-    breaker.isActive(100);
-    expect(getPayloadRevealCounts).toHaveBeenCalledWith(100 - faultInspectionWindow, 99);
+    const {breaker, getCanonicalPayloadCounts} = setup({full: 32, empty: 0});
+    breaker.isActive(100, head);
+    expect(getCanonicalPayloadCounts).toHaveBeenCalledWith(100 - faultInspectionWindow, 99, head);
   });
 
   it("requires a minimum sample to deactivate", () => {
-    const {breaker, getPayloadRevealCounts} = setup({blocksPresent: 1, payloadsRevealed: 0});
-    expect(breaker.isActive(100)).toBe(true);
+    const {breaker, getCanonicalPayloadCounts} = setup({full: 0, empty: 1});
+    expect(breaker.isActive(100, head)).toBe(true);
 
-    getPayloadRevealCounts.mockReturnValue({blocksPresent: 0, payloadsRevealed: 0});
-    expect(breaker.isActive(101)).toBe(true);
+    getCanonicalPayloadCounts.mockReturnValue({full: 0, empty: 0});
+    expect(breaker.isActive(101, head)).toBe(true);
 
-    getPayloadRevealCounts.mockReturnValue({blocksPresent: 3, payloadsRevealed: 3});
-    expect(breaker.isActive(102)).toBe(true);
+    getCanonicalPayloadCounts.mockReturnValue({full: 3, empty: 0});
+    expect(breaker.isActive(102, head)).toBe(true);
 
-    getPayloadRevealCounts.mockReturnValue({blocksPresent: 4, payloadsRevealed: 3});
-    expect(breaker.isActive(103)).toBe(false);
+    getCanonicalPayloadCounts.mockReturnValue({full: 3, empty: 1});
+    expect(breaker.isActive(103, head)).toBe(false);
   });
 
   it("only updates once per slot", () => {
-    const {breaker, getPayloadRevealCounts} = setup({blocksPresent: 32, payloadsRevealed: 32});
-    expect(breaker.isActive(100)).toBe(false);
+    const {breaker, getCanonicalPayloadCounts} = setup({full: 32, empty: 0});
+    expect(breaker.isActive(100, head)).toBe(false);
 
-    getPayloadRevealCounts.mockReturnValue({blocksPresent: 32, payloadsRevealed: 0});
-    expect(breaker.isActive(100)).toBe(false);
-    expect(getPayloadRevealCounts).toHaveBeenCalledTimes(1);
+    getCanonicalPayloadCounts.mockReturnValue({full: 0, empty: 32});
+    expect(breaker.isActive(100, head)).toBe(false);
+    expect(getCanonicalPayloadCounts).toHaveBeenCalledTimes(1);
 
-    expect(breaker.isActive(101)).toBe(true);
-    expect(getPayloadRevealCounts).toHaveBeenCalledTimes(2);
+    expect(breaker.isActive(101, head)).toBe(true);
+    expect(getCanonicalPayloadCounts).toHaveBeenCalledTimes(2);
   });
 
   describe("getFaultInspectionParams", () => {

packages/beacon-node/test/unit/chain/prepareNextSlot.test.ts

@@ -180,8 +180,10 @@ describe("PrepareNextSlot scheduler", () => {
 
   it("gloas - should update builder circuit breaker instead of builder status", async () => {
     getForkStub.mockReturnValue(ForkName.gloas);
-    chainStub.recomputeForkChoiceHead.mockReturnValue({...zeroProtoBlock, slot: SLOTS_PER_EPOCH - 3} as ProtoBlock);
-    chainStub.predictProposerHead.mockReturnValue({...zeroProtoBlock, slot: SLOTS_PER_EPOCH - 3} as ProtoBlock);
+    const headBlock = {...zeroProtoBlock, blockRoot: "0xhead", slot: SLOTS_PER_EPOCH - 3} as ProtoBlock;
+    const proposerHead = {...zeroProtoBlock, blockRoot: "0xparent", slot: SLOTS_PER_EPOCH - 4} as ProtoBlock;
+    chainStub.recomputeForkChoiceHead.mockReturnValue(headBlock);
+    chainStub.predictProposerHead.mockReturnValue(proposerHead);
     forkChoiceStub.getFinalizedBlock.mockReturnValue({} as ProtoBlock);
     const state = generateCachedBellatrixState();
     vi.spyOn(state.epochCtx, "getBeaconProposer").mockReturnValue(proposerIndex);
@@ -196,7 +198,7 @@ describe("PrepareNextSlot scheduler", () => {
       vi.advanceTimersByTimeAsync((config.SLOT_DURATION_MS * 2) / 3),
     ]);
 
-    expect(chainStub.builderCircuitBreaker.update).toHaveBeenCalledWith(SLOTS_PER_EPOCH - 2);
+    expect(chainStub.builderCircuitBreaker.update).toHaveBeenCalledWith(SLOTS_PER_EPOCH - 2, proposerHead);
     expect(updateBuilderStatus).not.toHaveBeenCalled();
   });
 });

packages/cli/src/options/beaconNodeOptions/builder.ts

@@ -55,14 +55,14 @@ export const options: CliCommandOptions<ExecutionBuilderArgs> = {
   "builder.faultInspectionWindow": {
     type: "number",
     description:
-      "Window to inspect missed slots (pre-gloas) or unrevealed payloads (post-gloas) for enabling/disabling builder circuit breaker",
+      "Window used to inspect missed slots (pre-gloas) or calculate the canonical EMPTY block rate (post-gloas) for enabling/disabling the builder circuit breaker",
     group: "builder",
   },
 
   "builder.allowedFaults": {
     type: "number",
     description:
-      "Number of missed slots allowed within `faultInspectionWindow` before ignoring the external builder (pre-gloas). Post-gloas, sets the tolerated rate of unrevealed payloads, defined as `allowedFaults` out of `faultInspectionWindow` and applied to blocks observed in the window",
+      "Number of missed slots allowed within `faultInspectionWindow` before ignoring the external builder (pre-gloas). Post-gloas, sets the tolerated rate of canonical EMPTY blocks, defined as `allowedFaults` out of `faultInspectionWindow` and applied to canonical blocks in the window",
     group: "builder",
   },
 };

packages/fork-choice/src/forkChoice/forkChoice.ts

@@ -626,8 +626,8 @@ export class ForkChoice implements IForkChoice {
     return this.protoArray.nodes.filter((node) => node.slot > windowStart).length;
   }
 
-  getPayloadRevealCounts(fromSlot: Slot, toSlot: Slot): {blocksPresent: number; payloadsRevealed: number} {
-    return this.protoArray.getPayloadRevealCounts(fromSlot, toSlot);
+  getCanonicalPayloadCounts(fromSlot: Slot, toSlot: Slot, head: ProtoBlock): {full: number; empty: number} {
+    return this.protoArray.getCanonicalPayloadCounts(fromSlot, toSlot, head.blockRoot, head.payloadStatus);
   }
 
   /** Very expensive function, iterates the entire ProtoArray. Called only in debug API */

packages/fork-choice/src/forkChoice/interface.ts

@@ -249,11 +249,8 @@ export interface IForkChoice {
   hasPayloadUnsafe(blockRoot: Root): boolean;
   hasPayloadHexUnsafe(blockRoot: RootHex): boolean;
   getSlotsPresent(windowStart: number): number;
-  /**
-   * Count Gloas blocks with fromSlot <= slot <= toSlot, and how many of them have a revealed payload
-   * (FULL variant exists). Used by the builder circuit breaker.
-   */
-  getPayloadRevealCounts(fromSlot: Slot, toSlot: Slot): {blocksPresent: number; payloadsRevealed: number};
+  /** Count FULL and EMPTY blocks on the supplied head chain in the inclusive slot range. */
+  getCanonicalPayloadCounts(fromSlot: Slot, toSlot: Slot, head: ProtoBlock): {full: number; empty: number};
   getPTCVotes(blockRootHex: RootHex): (boolean | null)[] | null;
   /** Raw PTC vote tallies for the debug fork choice endpoint; `null` for pre-Gloas roots. */
   getPTCVoteCounts(blockRootHex: RootHex): {

packages/fork-choice/src/protoArray/protoArray.ts

@@ -690,30 +690,42 @@ export class ProtoArray {
     this.maybeUpdateBestChildAndDescendant(pendingIndex, fullIndex, currentSlot, proposerBoostRoot);
   }
 
-  /**
-   * Count Gloas blocks with fromSlot <= slot <= toSlot, and how many of them have a revealed payload
-   * (FULL variant exists). Used by the builder circuit breaker.
-   */
-  getPayloadRevealCounts(fromSlot: Slot, toSlot: Slot): {blocksPresent: number; payloadsRevealed: number} {
-    let blocksPresent = 0;
-    let payloadsRevealed = 0;
-    // Full scan, nodes are in import order not slot order (an old block can be imported after newer
-    // ones during sync or reorg resolution), so we cannot stop early on an out-of-window slot
-    for (const node of this.nodes) {
-      // Count each gloas block once via its PENDING variant, pre-gloas nodes are FULL only
-      if (node.slot < fromSlot || node.slot > toSlot || node.payloadStatus !== PayloadStatus.PENDING) {
-        continue;
-      }
-      // Genesis block is always EMPTY
-      if (node.slot === GENESIS_SLOT) {
-        continue;
-      }
-      blocksPresent++;
-      if (this.hasPayload(node.blockRoot)) {
-        payloadsRevealed++;
+  /** Count blocks selected as FULL or EMPTY by the supplied head chain in the inclusive slot range. */
+  getCanonicalPayloadCounts(
+    fromSlot: Slot,
+    toSlot: Slot,
+    headRoot: RootHex,
+    headPayloadStatus: PayloadStatus
+  ): {full: number; empty: number} {
+    let full = 0;
+    let empty = 0;
+
+    // Walk the canonical chain newest-first from the head, following ancestors via `getParentNodeIndex`.
+    // Ancestors are strictly slot-descending, so we stop as soon as a node falls below `fromSlot`
+    // instead of materializing the whole chain back to the anchor as `getAllAncestorNodes` does.
+    // This keeps the scan O(window) rather than O(chain-to-anchor), relevant under prolonged non-finality.
+    const headIndex = this.getNodeIndexByRootAndStatus(headRoot, headPayloadStatus);
+    let node = headIndex !== undefined ? this.nodes[headIndex] : undefined;
+
+    while (node !== undefined && node.slot >= fromSlot) {
+      if (
+        node.slot !== GENESIS_SLOT &&
+        node.slot <= toSlot &&
+        isGloasBlock(node) &&
+        node.payloadStatus !== PayloadStatus.PENDING
+      ) {
+        if (node.payloadStatus === PayloadStatus.FULL) {
+          full++;
+        } else {
+          empty++;
+        }
       }
+
+      const parentIndex = this.getParentNodeIndex(node);
+      node = parentIndex === undefined ? undefined : this.nodes[parentIndex];
     }
-    return {blocksPresent, payloadsRevealed};
+
+    return {full, empty};
   }
 
   /**