feat: state cache for ePBS #8868

nflaig · 2026-03-18T12:58:43Z

more of a question, do we actually wanna persist full states? since for checkpoint sync we always wanna use post block state (without payload applied) what's the use case for this?

instead of storing 2 states we could also just load post block state and apply payload if needed

yes that could be a good improvement
right now in in checkpoint state cache it has no context of payload
whenever it goes out of memory windows (of 3 epochs), it persists whatever it has

the down side of not persisting payload state is when we have a finalized checkpoint of > 3 epochs ago, we may not have the state, we have to load post block Uint8Array state, deserialize and apply payload

ensi321 · 2026-03-19T05:46:36Z

If we have a FULL node in fork choice but we don't have payload in db.executionPayloadEnvelope, it sounds like an error not debug to me?

we have a convention to make it debug to improve UX, users don't care this much

agree, this doesn't need to be logged at error, there isn't much a user can do in this case, it's also an invariant that should not happen

-Original file line number
+Diff line change
@@ Expand Up @@
             meta: ExecutionOptimisticFinalizedAndVersionCodec,
           },
         },
+        // TODO GLOAS: this endpoint needs to be updated because post-gloas there could be two variants of the persisted checkpoint state (empty or full).
+        // Either add a an additional parameter `payloadPresent`, or return one or both variants of state.
         getPersistedCheckpointState: {
           url: "/eth/v1/lodestar/persisted_checkpoint_state",
           method: "GET",
@@ Expand Down @@

-Original file line number
+Diff line change
@@ -1,5 +1,5 @@
     import {routes} from "@lodestar/api";
-    import {CheckpointWithHex, IForkChoice} from "@lodestar/fork-choice";
+    import {CheckpointWithPayloadStatus, IForkChoice} from "@lodestar/fork-choice";
     import {GENESIS_SLOT} from "@lodestar/params";
     import {BeaconStateAllForks, CachedBeaconStateAllForks, PubkeyCache} from "@lodestar/state-transition";
     import {
@@ Expand All / @@ -19,7 +19,7 @@ import {ApiError, ValidationError} from "../../errors.js"; @@
     export function resolveStateId(
       forkChoice: IForkChoice,
       stateId: routes.beacon.StateId
-    ): RootHex | Slot | CheckpointWithHex {
+    ): RootHex | Slot | CheckpointWithPayloadStatus {
       if (stateId === "head") {
         return forkChoice.getHead().stateRoot;
       }
@@ Expand Down @@

-Original file line number
+Diff line change
@@ Expand Up @@
     import {PREPARE_NEXT_SLOT_BPS} from "../../../chain/prepareNextSlot.js";
     import {BlockType, ProduceFullDeneb, ProduceFullGloas} from "../../../chain/produceBlock/index.js";
     import {RegenCaller} from "../../../chain/regen/index.js";
-    import {CheckpointHex} from "../../../chain/stateCache/types.js";
+    import {CheckpointHexPayload} from "../../../chain/stateCache/types.js";
     import {validateApiAggregateAndProof} from "../../../chain/validation/index.js";
     import {validateSyncCommitteeGossipContributionAndProof} from "../../../chain/validation/syncCommitteeContributionAndProof.js";
     import {ZERO_HASH} from "../../../constants/index.js";
@@ Expand Down Expand Up / @@ -303,7 +303,7 @@ export function getValidatorApi( @@
        *                    |
        *              prepareNextSlot (4s before next slot)
        */
-      async function waitForCheckpointState(cpHex: CheckpointHex): Promise<CachedBeaconStateAllForks | null> {
+      async function waitForCheckpointState(cpHex: CheckpointHexPayload): Promise<CachedBeaconStateAllForks | null> {
         const cpState = chain.regen.getCheckpointStateSync(cpHex);
         if (cpState) {
           return cpState;
@@ Expand Down Expand Up / @@ -1112,7 +1112,11 @@ export function getValidatorApi( @@
           // this is to avoid missed block proposal due to 0 epoch look ahead
           if (epoch === nextEpoch && toNextEpochMs < prepareNextSlotLookAheadMs) {
             // wait for maximum 1 slot for cp state which is the timeout of validator api
-            const cpState = await waitForCheckpointState({rootHex: head.blockRoot, epoch});
+            const cpState = await waitForCheckpointState({
+              rootHex: head.blockRoot,
+              epoch,
+              payloadPresent: head.payloadStatus === PayloadStatus.FULL,
+            });
             if (cpState) {
               state = cpState;
               metrics?.duties.requestNextEpochProposalDutiesHit.inc();
@@ Expand Down @@

-Original file line number
+Diff line change
@@ -1,6 +1,5 @@
     import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice";
     import {LoggerNode} from "@lodestar/logger/node";
-    import {ForkSeq} from "@lodestar/params";
     import {Checkpoint} from "@lodestar/types/phase0";
     import {callFnWhenAwait} from "@lodestar/utils";
     import {IBeaconDb} from "../../db/index.js";
@@ Expand All @@
     import {ArchiveMode, ArchiveStoreOpts, StateArchiveStrategy} from "./interface.js";
     import {FrequencyStateArchiveStrategy} from "./strategies/frequencyStateArchiveStrategy.js";
     import {archiveBlocks} from "./utils/archiveBlocks.js";
-    import {archiveExecutionPayloadEnvelopes} from "./utils/archivePayloads.js";
     import {pruneHistory} from "./utils/pruneHistory.js";
     import {updateBackfillRange} from "./utils/updateBackfillRange.js";
@@ Expand All @@
     export enum ArchiveStoreTask {
       ArchiveBlocks = "archive_blocks",
-      ArchivePayloads = "archive_payloads",
       PruneHistory = "prune_history",
       OnFinalizedCheckpoint = "on_finalized_checkpoint",
       MaybeArchiveState = "maybe_archive_state",
@@ Expand Down Expand Up / @@ -192,7 +189,6 @@ export class ArchiveStore { @@
       private processFinalizedCheckpoint = async (finalized: CheckpointWithPayloadStatus): Promise<void> => {
         try {
           const finalizedEpoch = finalized.epoch;
-          const finalizedFork = this.chain.config.getForkSeqAtEpoch(finalizedEpoch);
           this.logger.verbose("Start processing finalized checkpoint", {epoch: finalizedEpoch, rootHex: finalized.rootHex});
           let timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer();
@@ Expand All / @@ -210,12 +206,6 @@ export class ArchiveStore { @@
           );
           timer?.({source: ArchiveStoreTask.ArchiveBlocks});
-          if (finalizedFork >= ForkSeq.gloas) {
-            timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer();
-            await archiveExecutionPayloadEnvelopes(this.chain, finalized);
-            timer?.({source: ArchiveStoreTask.ArchivePayloads});
-          }
           if (this.opts.pruneHistory) {
             timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer();
             await pruneHistory(
@@ Expand Down @@

-Original file line number
+Diff line change
@@ -1,4 +1,4 @@
-    import {CheckpointWithHex} from "@lodestar/fork-choice";
+    import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice";
     import {RootHex} from "@lodestar/types";
     import {Metrics} from "../../metrics/metrics.js";
@@ Expand Down Expand Up / @@ -44,9 +44,9 @@ export type FinalizedStats = { @@
     export interface StateArchiveStrategy {
       onCheckpoint(stateRoot: RootHex, metrics?: Metrics | null): Promise<void>;
-      onFinalizedCheckpoint(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
-      maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
-      archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
+      onFinalizedCheckpoint(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise<void>;
+      maybeArchiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise<void>;
+      archiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise<void>;
     }
     export interface IArchiveStore {
@@ Expand Down @@

Navigation Menu

Uh oh!

Uh oh!

feat: state cache for ePBS #8868

Uh oh!

Diff view

Diff view

There are no files selected for viewing

nflaig Mar 18, 2026

Uh oh!

twoeths Mar 19, 2026

Uh oh!

Uh oh!

ensi321 Mar 19, 2026

Uh oh!

twoeths Mar 19, 2026

Uh oh!

nflaig Mar 19, 2026

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

-Original file line number
+Diff line change
@@ -1,4 +1,4 @@
-    import {CheckpointWithHex} from "@lodestar/fork-choice";
+    import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice";
     import {SLOTS_PER_EPOCH} from "@lodestar/params";
     import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
     import {Epoch, RootHex, Slot} from "@lodestar/types";
@@ Expand All @@
     import {getStateSlotFromBytes} from "../../../util/multifork.js";
     import {IStateRegenerator} from "../../regen/interface.js";
     import {serializeState} from "../../serializeState.js";
+    import {fcCheckpointToHexPayload} from "../../stateCache/persistentCheckpointsCache.js";
     import {StateArchiveStrategy, StatesArchiveOpts} from "../interface.js";
     /**
@@ Expand Down Expand Up @@
         private readonly bufferPool?: BufferPool | null
       ) {}
-      async onFinalizedCheckpoint(_finalized: CheckpointWithHex, _metrics?: Metrics | null): Promise<void> {}
+      async onFinalizedCheckpoint(_finalized: CheckpointWithPayloadStatus, _metrics?: Metrics | null): Promise<void> {}
       async onCheckpoint(_stateRoot: RootHex, _metrics?: Metrics | null): Promise<void> {}
       /**
@@ Expand All @@
        * epoch - 1024*2    epoch - 1024    epoch - 32    epoch
        * ```
        */
-      async maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void> {
+      async maybeArchiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise<void> {
         let timer = metrics?.processFinalizedCheckpoint.frequencyStateArchive.startTimer();
         const lastStoredSlot = await this.db.stateArchive.lastKey();
         timer?.({step: FrequencyStateArchiveStep.LoadLastStoredSlot});
@@ Expand Down Expand Up @@
        * Archives finalized states from active bucket to archive bucket.
        * Only the new finalized state is stored to disk
        */
-      async archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void> {
+      async archiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise<void> {
         // starting from Mar 2024, the finalized state could be from disk or in memory
         let timer = metrics?.processFinalizedCheckpoint.frequencyStateArchive.startTimer();
-        const finalizedStateOrBytes = await this.regen.getCheckpointStateOrBytes(finalized);
+        // Convert fork-choice checkpoint to beacon-node checkpoint with payloadPresent
+        const finalizedHexPayload = fcCheckpointToHexPayload(finalized);
+        const finalizedStateOrBytes = await this.regen.getCheckpointStateOrBytes(finalizedHexPayload);
         timer?.({step: FrequencyStateArchiveStep.GetFinalizedState});
         const {rootHex} = finalized;
@@ Expand Down @@

-Original file line number
+Diff line change
@@ -1,7 +1,7 @@
     import path from "node:path";
     import {ChainForkConfig} from "@lodestar/config";
     import {KeyValue} from "@lodestar/db";
-    import {CheckpointWithPayloadStatus, IForkChoice} from "@lodestar/fork-choice";
+    import {CheckpointWithPayloadStatus, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
     import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
     import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
     import {Epoch, Slot} from "@lodestar/types";
@@ Expand Down Expand Up / @@ -66,6 +66,7 @@ export async function archiveBlocks( @@
       // NOTE: The finalized block will be exactly the first block of `epoch` or previous
       const finalizedPostDeneb = finalizedCheckpoint.epoch >= config.DENEB_FORK_EPOCH;
       const finalizedPostFulu = finalizedCheckpoint.epoch >= config.FULU_FORK_EPOCH;
+      const finalizedPostGloas = finalizedCheckpoint.epoch >= config.GLOAS_FORK_EPOCH;
       const finalizedCanonicalBlockRoots: BlockRootSlot[] = finalizedCanonicalBlocks.map((block) => ({
         slot: block.slot,
@@ Expand Down Expand Up / @@ -103,6 +104,16 @@ export async function archiveBlocks( @@
           );
           logger.verbose("Migrated dataColumnSidecars from hot DB to cold DB", {...logCtx, migratedEntries});
         }
+        if (finalizedPostGloas) {
+          const migratedEntries = await migrateExecutionPayloadEnvelopesFromHotToColdDb(
+            config,
+            db,
+            logger,
+            finalizedCanonicalBlocks
+          );
+          logger.verbose("Migrated executionPayloadEnvelopes from hot DB to cold DB", {...logCtx, migratedEntries});
+        }
       }
       // deleteNonCanonicalBlocks
@@ Expand Down Expand Up / @@ -144,6 +155,11 @@ export async function archiveBlocks( @@
           await db.dataColumnSidecar.deleteMany(nonCanonicalBlockRoots);
           logger.verbose("Deleted non canonical dataColumnSidecars from hot DB", logCtx);
         }
+        if (finalizedPostGloas) {
+          await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots);
+          logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", logCtx);
+        }
       }
       // Delete expired blobs
@@ Expand Down Expand Up / @@ -372,6 +388,48 @@ async function migrateDataColumnSidecarsFromHotToColdDb( @@
       return migratedWrappedDataColumns;
     }
+    async function migrateExecutionPayloadEnvelopesFromHotToColdDb(
+      config: ChainForkConfig,
+      db: IBeaconDb,
+      logger: Logger,
+      canonicalBlocks: ProtoBlock[]
+    ): Promise<number> {
+      let migratedEnvelopes = 0;
+      const payloadBlocks = canonicalBlocks.filter(
+        (block) => config.getForkSeq(block.slot) >= ForkSeq.gloas && block.payloadStatus === PayloadStatus.FULL
+      );
+      if (payloadBlocks.length === 0) return 0;
+      const blocks = payloadBlocks.map((block) => ({slot: block.slot, root: fromHex(block.blockRoot)}));
+      const envelopeEntries: KeyValue<Slot, Uint8Array>[] = [];
+      const migratedRoots: Uint8Array[] = [];
+      const envelopeBytesArray = await Promise.all(
+        blocks.map((block) => db.executionPayloadEnvelope.getBinary(block.root))
+      );
+      for (let i = 0; i < blocks.length; i++) {
+        const bytes = envelopeBytesArray[i];
+        if (bytes !== null) {
+          envelopeEntries.push({key: blocks[i].slot, value: bytes});
+          migratedRoots.push(blocks[i].root);
+        } else {
+          logger.debug("Payload in forkchoice but missing in db", {slot: blocks[i].slot, root: toRootHex(blocks[i].root)});
+        }
+      }
+      if (envelopeEntries.length > 0) {
+        await Promise.all([
+          db.executionPayloadEnvelopeArchive.batchPutBinary(envelopeEntries),
+          db.executionPayloadEnvelope.batchDelete(migratedRoots),
+        ]);
+        migratedEnvelopes = envelopeEntries.length;
+      }
+      return migratedEnvelopes;
+    }
     /**
      * ```
      * class SignedBeaconBlock(Container):
@@ Expand Down @@

-Original file line number
+Diff line change
@@ Expand Up / @@ -7,6 +7,7 @@ import { @@
       ForkChoiceErrorCode,
       NotReorgedReason,
       getSafeExecutionBlockHash,
+      isGloasBlock,
     } from "@lodestar/fork-choice";
     import {ForkPostAltair, ForkPostElectra, ForkSeq, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params";
     import {
@@ Expand All / @@ -30,7 +31,7 @@ import type {BeaconChain} from "../chain.js"; @@
     import {ChainEvent, ReorgEventData} from "../emitter.js";
     import {ForkchoiceCaller} from "../forkChoice/index.js";
     import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "../reprocess.js";
-    import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js";
+    import {toCheckpointHexPayload} from "../stateCache/persistentCheckpointsCache.js";
     import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js";
     import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
     import {getCheckpointFromState} from "./utils/checkpoint.js";
@@ Expand Down Expand Up / @@ -116,7 +117,11 @@ export async function importBlock( @@
       // This adds the state necessary to process the next block
       // Some block event handlers require state being in state cache so need to do this before emitting EventType.block
-      this.regen.processState(blockRootHex, postState);
+      // Pre-Gloas: blockSummary.payloadStatus is always FULL, payloadPresent = true
+      // Post-Gloas: blockSummary.payloadStatus is always PENDING, so payloadPresent = false (block state only, no payload processing yet)
+      const payloadPresent = !isGloasBlock(blockSummary);
+      // processState manages both block state and payload state variants together for memory/disk management
+      this.regen.processBlockState(blockRootHex, postState);
       this.metrics?.importBlock.bySource.inc({source: source.source});
       this.logger.verbose("Added block to forkchoice and state cache", {slot: blockSlot, root: blockRootHex});
@@ Expand Down Expand Up / @@ -456,12 +461,12 @@ export async function importBlock( @@
         // Cache state to preserve epoch transition work
         const checkpointState = postState;
         const cp = getCheckpointFromState(checkpointState);
-        this.regen.addCheckpointState(cp, checkpointState);
+        this.regen.addCheckpointState(cp, checkpointState, payloadPresent);
         // consumers should not mutate state ever
         this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState);
         // Note: in-lined code from previos handler of ChainEvent.checkpoint
-        this.logger.verbose("Checkpoint processed", toCheckpointHex(cp));
+        this.logger.verbose("Checkpoint processed", toCheckpointHexPayload(cp, payloadPresent));
         const activeValidatorsCount = checkpointState.epochCtx.currentShuffling.activeIndices.length;
         this.metrics?.currentActiveValidators.set(activeValidatorsCount);
@@ Expand All / @@ -479,7 +484,7 @@ export async function importBlock( @@
           const justifiedEpoch = justifiedCheckpoint.epoch;
           const preJustifiedEpoch = parentBlockSummary.justifiedEpoch;
           if (justifiedEpoch > preJustifiedEpoch) {
-            this.logger.verbose("Checkpoint justified", toCheckpointHex(justifiedCheckpoint));
+            this.logger.verbose("Checkpoint justified", toCheckpointHexPayload(justifiedCheckpoint, payloadPresent));
             this.metrics?.previousJustifiedEpoch.set(checkpointState.previousJustifiedCheckpoint.epoch);
             this.metrics?.currentJustifiedEpoch.set(justifiedCheckpoint.epoch);
           }
@@ Expand All / @@ -493,7 +498,7 @@ export async function importBlock( @@
               state: toRootHex(checkpointState.hashTreeRoot()),
               executionOptimistic: false,
             });
-            this.logger.verbose("Checkpoint finalized", toCheckpointHex(finalizedCheckpoint));
+            this.logger.verbose("Checkpoint finalized", toCheckpointHexPayload(finalizedCheckpoint, payloadPresent));
             this.metrics?.finalizedEpoch.set(finalizedCheckpoint.epoch);
           }
         }
@@ Expand Down @@

Uh oh!

Uh oh!

feat: state cache for ePBS #8868

Uh oh!

feat: state cache for ePBS #8868

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

nflaig Mar 18, 2026

Choose a reason for hiding this comment

Uh oh!

twoeths Mar 19, 2026

Choose a reason for hiding this comment

Uh oh!

Uh oh!

ensi321 Mar 19, 2026

Choose a reason for hiding this comment

Uh oh!

twoeths Mar 19, 2026

Choose a reason for hiding this comment

Uh oh!

nflaig Mar 19, 2026

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!