feat: add circuit breaker for gloas block production

.wordlist.txt

@@ -169,6 +169,7 @@ flamegraphs
 floodsub
 fsSL
 getNetworkIdentity
+gloas
 gnosis
 gpg
 heapdump

packages/beacon-node/src/api/impl/validator/index.ts

@@ -938,14 +938,18 @@ export function getValidatorApi(
       // support when it is implemented.
       const isBuildingOnFull = chain.forkChoice.shouldBuildOnFull(parentBlock, slot);
       const bidParentBlockHash = isBuildingOnFull ? parentBlock.executionPayloadBlockHash : parentBlock.parentBlockHash;
-      const builderBid = chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
+      const circuitBreakerActive = chain.builderCircuitBreaker.isActive(slot);
+      const builderBid = circuitBreakerActive
+        ? null
+        : chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
 
       const logCtx = {
         slot,
         parentSlot,
         parentBlockRoot: parentBlockRootHex,
         parentBlockHash: parentBlock.executionPayloadBlockHash,
         fork,
+        circuitBreakerActive,
         ...(builderBid !== null
           ? {
               bidValue: builderBid.message.value,

packages/beacon-node/src/chain/builderCircuitBreaker.ts

@@ -0,0 +1,84 @@
+import {IForkChoice} from "@lodestar/fork-choice";
+import {Slot} from "@lodestar/types";
+import {Logger} from "@lodestar/utils";
+import {getFaultInspectionParams} from "../execution/builder/http.js";
+import {Metrics} from "../metrics/index.js";
+
+export type BuilderCircuitBreakerOpts = {
+  faultInspectionWindow?: number;
+  allowedFaults?: number;
+};
+
+export type BuilderCircuitBreakerModules = {
+  forkChoice: IForkChoice;
+  logger: Logger;
+  metrics: Metrics | null;
+};
+
+/**
+ * Post-gloas circuit breaker for builder bids. The beacon block is produced by the proposer
+ * regardless of bid source, so missed blocks are not a useful builder health signal. Instead
+ * count blocks whose payload was never revealed and stop selecting builder bids while the
+ * non-reveal rate in the fault inspection window is too high.
+ */
+export class BuilderCircuitBreaker {
+  readonly faultInspectionWindow: number;
+  readonly allowedFaults: number;
+
+  private active = false;
+  private lastUpdatedSlot = -1;
+
+  constructor(
+    opts: BuilderCircuitBreakerOpts,
+    private readonly modules: BuilderCircuitBreakerModules
+  ) {
+    const {faultInspectionWindow, allowedFaults} = getFaultInspectionParams(opts);
+    this.faultInspectionWindow = faultInspectionWindow;
+    this.allowedFaults = allowedFaults;
+  }
+
+  /** Whether builder bids must be ignored for a block produced at clockSlot */
+  isActive(clockSlot: Slot): boolean {
+    this.update(clockSlot);
+    return this.active;
+  }
+
+  update(clockSlot: Slot): void {
+    if (clockSlot <= this.lastUpdatedSlot) {
+      return;
+    }
+    this.lastUpdatedSlot = clockSlot;
+
+    // Exclude clockSlot itself, its payload reveal may still be in flight
+    const {blocksPresent, payloadsRevealed} = this.modules.forkChoice.getPayloadRevealCounts(
+      Math.max(clockSlot - this.faultInspectionWindow, 0),
+      clockSlot - 1
+    );
+    const faults = blocksPresent - payloadsRevealed;
+
+    const wasActive = this.active;
+    // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates
+    this.active = faults * this.faultInspectionWindow > this.allowedFaults * blocksPresent;
+
+    this.modules.metrics?.builderCircuitBreaker.active.set(this.active ? 1 : 0);
+    this.modules.metrics?.builderCircuitBreaker.faults.set(faults);
+    this.modules.metrics?.builderCircuitBreaker.blocksPresent.set(blocksPresent);
+    this.modules.metrics?.builderCircuitBreaker.payloadsRevealed.set(payloadsRevealed);
+
+    const logCtx = {
+      blocksPresent,
+      faults,
+      faultInspectionWindow: this.faultInspectionWindow,
+      allowedFaults: this.allowedFaults,
+    };
+    if (this.active !== wasActive) {
+      if (this.active) {
+        this.modules.logger.warn("Builder circuit breaker activated, ignoring builder bids", logCtx);
+      } else {
+        this.modules.logger.info("Builder circuit breaker deactivated", logCtx);
+      }
+    } else {
+      this.modules.logger.verbose("Builder circuit breaker status", {active: this.active, ...logCtx});
+    }
+  }
+}

packages/beacon-node/src/chain/chain.ts

@@ -75,6 +75,7 @@ import {ImportPayloadOpts} from "./blocks/types.js";
 import {persistBlockInput} from "./blocks/writeBlockInputToDb.js";
 import {persistPayloadEnvelopeInput} from "./blocks/writePayloadEnvelopeInputToDb.js";
 import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier, IBlsVerifier} from "./bls/index.js";
+import {BuilderCircuitBreaker} from "./builderCircuitBreaker.js";
 import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js";
 import {ChainEvent, ChainEventEmitter} from "./emitter.js";
 import {ForkchoiceCaller, initializeForkChoice} from "./forkChoice/index.js";
@@ -151,6 +152,7 @@ export class BeaconChain implements IBeaconChain {
   readonly genesisValidatorsRoot: Root;
   readonly executionEngine: IExecutionEngine;
   readonly executionBuilder?: IExecutionBuilder;
+  readonly builderCircuitBreaker: BuilderCircuitBreaker;
   // Expose config for convenience in modularized functions
   readonly config: BeaconConfig;
   readonly custodyConfig: CustodyConfig;
@@ -425,6 +427,11 @@ export class BeaconChain implements IBeaconChain {
 
     this.forkChoice = forkChoice;
 
+    this.builderCircuitBreaker = new BuilderCircuitBreaker(
+      {faultInspectionWindow: opts.faultInspectionWindow, allowedFaults: opts.allowedFaults},
+      {forkChoice, logger, metrics}
+    );
+
     this.seenPayloadEnvelopeInputCache = new SeenPayloadEnvelopeInput({
       config,
       clock,

packages/beacon-node/src/chain/interface.ts

@@ -35,6 +35,7 @@ import {BeaconProposerCache, ProposerPreparationData} from "./beaconProposerCach
 import {IBlockInput} from "./blocks/blockInput/index.js";
 import {ImportBlockOpts, ImportPayloadOpts} from "./blocks/types.js";
 import {IBlsVerifier} from "./bls/index.js";
+import {BuilderCircuitBreaker} from "./builderCircuitBreaker.js";
 import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js";
 import {ChainEventEmitter} from "./emitter.js";
 import {ForkchoiceCaller} from "./forkChoice/index.js";
@@ -95,6 +96,7 @@ export interface IBeaconChain {
   readonly earliestAvailableSlot: Slot;
   readonly executionEngine: IExecutionEngine;
   readonly executionBuilder?: IExecutionBuilder;
+  readonly builderCircuitBreaker: BuilderCircuitBreaker;
   // Expose config for convenience in modularized functions
   readonly config: BeaconConfig;
   readonly custodyConfig: CustodyConfig;

packages/beacon-node/src/chain/options.ts

@@ -49,6 +49,10 @@ export type IChainOptions = BlockProcessOpts &
     archiveDateEpochs?: number;
     nHistoricalStatesFileDataStore?: boolean;
     nativeStateView?: boolean;
+    /** Builder circuit breaker fault inspection window in slots */
+    faultInspectionWindow?: number;
+    /** Allowed unrevealed payloads within the fault inspection window */
+    allowedFaults?: number;
   };
 
 export type BlockProcessOpts = {

packages/beacon-node/src/chain/prepareNextSlot.ts

@@ -1,7 +1,7 @@
 import {routes} from "@lodestar/api";
 import {ChainForkConfig} from "@lodestar/config";
 import {getSafeExecutionBlockHash} from "@lodestar/fork-choice";
-import {ForkPostBellatrix, ForkSeq, SLOTS_PER_EPOCH, isForkPostBellatrix} from "@lodestar/params";
+import {ForkPostBellatrix, ForkSeq, SLOTS_PER_EPOCH, isForkPostBellatrix, isForkPostGloas} from "@lodestar/params";
 import {
   IBeaconStateView,
   IBeaconStateViewBellatrix,
@@ -152,12 +152,16 @@ export class PrepareNextSlotScheduler {
             updatedHead = proposerHead;
           }
 
-          // Update the builder status, if enabled shoot an api call to check status
-          this.chain.updateBuilderStatus(clockSlot);
-          if (this.chain.executionBuilder?.status === BuilderStatus.enabled) {
-            this.chain.executionBuilder.checkStatus().catch((e) => {
-              this.logger.error("Builder disabled as the check status api failed", {prepareSlot}, e as Error);
-            });
+          if (isForkPostGloas(fork)) {
+            this.chain.builderCircuitBreaker.update(clockSlot);
+          } else {
+            // Update the builder status, if enabled shoot an api call to check status
+            this.chain.updateBuilderStatus(clockSlot);
+            if (this.chain.executionBuilder?.status === BuilderStatus.enabled) {
+              this.chain.executionBuilder.checkStatus().catch((e) => {
+                this.logger.error("Builder disabled as the check status api failed", {prepareSlot}, e as Error);
+              });
+            }
           }
         }
 

packages/beacon-node/src/execution/builder/http.ts

@@ -70,6 +70,33 @@ export class NoBidReceived extends Error {
   }
 }
 
+/**
+ * Beacon clients select randomized values from the following ranges when initializing
+ * the circuit breaker (so at boot time and once for each unique boot).
+ *
+ * ALLOWED_FAULTS: between 1 and SLOTS_PER_EPOCH // 4
+ * FAULT_INSPECTION_WINDOW: between SLOTS_PER_EPOCH and 2 * SLOTS_PER_EPOCH
+ *
+ * The values are randomized per node so builders cannot predict when a given proposer will
+ * fall back to local blocks. With fixed thresholds a builder could withhold payloads right up
+ * to the limit without ever tripping the breaker.
+ */
+export function getFaultInspectionParams(opts: {faultInspectionWindow?: number; allowedFaults?: number}): {
+  faultInspectionWindow: number;
+  allowedFaults: number;
+} {
+  const faultInspectionWindow = Math.max(
+    opts.faultInspectionWindow ?? SLOTS_PER_EPOCH + Math.floor(Math.random() * SLOTS_PER_EPOCH),
+    SLOTS_PER_EPOCH
+  );
+  // allowedFaults should be < faultInspectionWindow, limiting them to faultInspectionWindow/4
+  const allowedFaults = Math.min(
+    opts.allowedFaults ?? Math.floor(faultInspectionWindow / 4),
+    Math.floor(faultInspectionWindow / 4)
+  );
+  return {faultInspectionWindow, allowedFaults};
+}
+
 /**
  * Additional duration to account for potential event loop lag which causes
  * builder blocks to be rejected even though the response was sent in time.
@@ -122,23 +149,9 @@ export class ExecutionBuilderHttp implements IExecutionBuilder {
     this.registrations = new ValidatorRegistrationCache();
     this.issueLocalFcUWithFeeRecipient = opts.issueLocalFcUWithFeeRecipient;
 
-    /**
-     * Beacon clients select randomized values from the following ranges when initializing
-     * the circuit breaker (so at boot time and once for each unique boot).
-     *
-     * ALLOWED_FAULTS: between 1 and SLOTS_PER_EPOCH // 4
-     * FAULT_INSPECTION_WINDOW: between SLOTS_PER_EPOCH and 2 * SLOTS_PER_EPOCH
-     *
-     */
-    this.faultInspectionWindow = Math.max(
-      opts.faultInspectionWindow ?? SLOTS_PER_EPOCH + Math.floor(Math.random() * SLOTS_PER_EPOCH),
-      SLOTS_PER_EPOCH
-    );
-    // allowedFaults should be < faultInspectionWindow, limiting them to faultInspectionWindow/4
-    this.allowedFaults = Math.min(
-      opts.allowedFaults ?? Math.floor(this.faultInspectionWindow / 4),
-      Math.floor(this.faultInspectionWindow / 4)
-    );
+    const {faultInspectionWindow, allowedFaults} = getFaultInspectionParams(opts);
+    this.faultInspectionWindow = faultInspectionWindow;
+    this.allowedFaults = allowedFaults;
   }
 
   updateStatus(status: BuilderStatus): void {

packages/beacon-node/src/metrics/metrics/beacon.ts

@@ -138,6 +138,25 @@ export function createBeaconMetrics(register: RegistryMetricCreator) {
       help: "Count of cached produced results",
     }),
 
+    builderCircuitBreaker: {
+      active: register.gauge({
+        name: "beacon_builder_circuit_breaker_active",
+        help: "Whether the builder circuit breaker is active (1) causing builder bids to be ignored",
+      }),
+      faults: register.gauge({
+        name: "beacon_builder_circuit_breaker_faults",
+        help: "Count of blocks with unrevealed payloads in the fault inspection window",
+      }),
+      blocksPresent: register.gauge({
+        name: "beacon_builder_circuit_breaker_blocks_present",
+        help: "Count of blocks present in the fault inspection window",
+      }),
+      payloadsRevealed: register.gauge({
+        name: "beacon_builder_circuit_breaker_payloads_revealed",
+        help: "Count of blocks with revealed payloads in the fault inspection window",
+      }),
+    },
+
     blockPayload: {
       payloadAdvancePrepTime: register.histogram({
         name: "beacon_block_payload_prepare_time",

packages/beacon-node/test/mocks/mockedBeaconChain.ts

@@ -5,9 +5,11 @@ import {EpochDifference, ForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {createPubkeyCache} from "@lodestar/state-transition";
 import {Logger} from "@lodestar/utils";
 import {BeaconProposerCache} from "../../src/chain/beaconProposerCache.js";
+import {BuilderCircuitBreaker} from "../../src/chain/builderCircuitBreaker.js";
 import {BeaconChain} from "../../src/chain/chain.js";
 import {ChainEventEmitter} from "../../src/chain/emitter.js";
 import {LightClientServer} from "../../src/chain/lightClient/index.js";
+import {ExecutionPayloadBidPool} from "../../src/chain/opPools/executionPayloadBidPool.js";
 import {AggregatedAttestationPool, OpPool, SyncContributionAndProofPool} from "../../src/chain/opPools/index.js";
 import {QueuedStateRegenerator} from "../../src/chain/regen/index.js";
 import {SeenBlockInput} from "../../src/chain/seenCache/seenGossipBlockInput.js";
@@ -23,6 +25,8 @@ export type MockedBeaconChain = Mocked<BeaconChain> & {
   forkChoice: MockedForkChoice;
   executionEngine: Mocked<ExecutionEngineHttp>;
   executionBuilder: Mocked<ExecutionBuilderHttp>;
+  builderCircuitBreaker: Mocked<BuilderCircuitBreaker>;
+  executionPayloadBidPool: Mocked<ExecutionPayloadBidPool>;
   opPool: Mocked<OpPool>;
   aggregatedAttestationPool: Mocked<AggregatedAttestationPool>;
   syncContributionAndProofPool: Mocked<SyncContributionAndProofPool>;
@@ -71,6 +75,8 @@ vi.mock("@lodestar/fork-choice", async (importActual) => {
       hasBlockHex: vi.fn(),
       getBlockSummariesAtSlot: vi.fn(),
       notifyPtcMessages: vi.fn(),
+      shouldBuildOnFull: vi.fn(),
+      getPayloadRevealCounts: vi.fn(),
     };
   });
 
@@ -143,6 +149,14 @@ vi.mock("../../src/chain/chain.js", async (importActual) => {
         getClientVersion: vi.fn(),
       },
       executionBuilder: {},
+      builderCircuitBreaker: {
+        isActive: vi.fn(),
+        update: vi.fn(),
+      },
+      executionPayloadBidPool: {
+        add: vi.fn(),
+        getBestBid: vi.fn(),
+      },
       opPool: new OpPool(config as BeaconConfig),
       aggregatedAttestationPool: new AggregatedAttestationPool(config as BeaconConfig),
       syncContributionAndProofPool: new SyncContributionAndProofPool(config, clock),