diff --git a/packages/api/src/beacon/routes/lodestar.ts b/packages/api/src/beacon/routes/lodestar.ts index 5236d3d40e38..ea894391f8f2 100644 --- a/packages/api/src/beacon/routes/lodestar.ts +++ b/packages/api/src/beacon/routes/lodestar.ts @@ -547,6 +547,8 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions { + async function waitForCheckpointState(cpHex: CheckpointHexPayload): Promise { const cpState = chain.regen.getCheckpointStateSync(cpHex); if (cpState) { return cpState; @@ -1112,7 +1112,11 @@ export function getValidatorApi( // this is to avoid missed block proposal due to 0 epoch look ahead if (epoch === nextEpoch && toNextEpochMs < prepareNextSlotLookAheadMs) { // wait for maximum 1 slot for cp state which is the timeout of validator api - const cpState = await waitForCheckpointState({rootHex: head.blockRoot, epoch}); + const cpState = await waitForCheckpointState({ + rootHex: head.blockRoot, + epoch, + payloadPresent: head.payloadStatus === PayloadStatus.FULL, + }); if (cpState) { state = cpState; metrics?.duties.requestNextEpochProposalDutiesHit.inc(); diff --git a/packages/beacon-node/src/chain/archiveStore/archiveStore.ts b/packages/beacon-node/src/chain/archiveStore/archiveStore.ts index 812f01034d53..b78330d76313 100644 --- a/packages/beacon-node/src/chain/archiveStore/archiveStore.ts +++ b/packages/beacon-node/src/chain/archiveStore/archiveStore.ts @@ -1,6 +1,5 @@ import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import {LoggerNode} from "@lodestar/logger/node"; -import {ForkSeq} from "@lodestar/params"; import {Checkpoint} from "@lodestar/types/phase0"; import {callFnWhenAwait} from "@lodestar/utils"; import {IBeaconDb} from "../../db/index.js"; @@ -14,7 +13,6 @@ import {HistoricalStateRegen} from "./historicalState/historicalStateRegen.js"; import {ArchiveMode, ArchiveStoreOpts, StateArchiveStrategy} from "./interface.js"; import {FrequencyStateArchiveStrategy} from "./strategies/frequencyStateArchiveStrategy.js"; import {archiveBlocks} from "./utils/archiveBlocks.js"; -import {archiveExecutionPayloadEnvelopes} from "./utils/archivePayloads.js"; import {pruneHistory} from "./utils/pruneHistory.js"; import {updateBackfillRange} from "./utils/updateBackfillRange.js"; @@ -29,7 +27,6 @@ type ArchiveStoreInitOpts = ArchiveStoreOpts & {dbName: string; anchorState: {fi export enum ArchiveStoreTask { ArchiveBlocks = "archive_blocks", - ArchivePayloads = "archive_payloads", PruneHistory = "prune_history", OnFinalizedCheckpoint = "on_finalized_checkpoint", MaybeArchiveState = "maybe_archive_state", @@ -192,7 +189,6 @@ export class ArchiveStore { private processFinalizedCheckpoint = async (finalized: CheckpointWithPayloadStatus): Promise => { try { const finalizedEpoch = finalized.epoch; - const finalizedFork = this.chain.config.getForkSeqAtEpoch(finalizedEpoch); this.logger.verbose("Start processing finalized checkpoint", {epoch: finalizedEpoch, rootHex: finalized.rootHex}); let timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer(); @@ -210,12 +206,6 @@ export class ArchiveStore { ); timer?.({source: ArchiveStoreTask.ArchiveBlocks}); - if (finalizedFork >= ForkSeq.gloas) { - timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer(); - await archiveExecutionPayloadEnvelopes(this.chain, finalized); - timer?.({source: ArchiveStoreTask.ArchivePayloads}); - } - if (this.opts.pruneHistory) { timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer(); await pruneHistory( diff --git a/packages/beacon-node/src/chain/archiveStore/interface.ts b/packages/beacon-node/src/chain/archiveStore/interface.ts index 67054c36463c..25b54c4fa38f 100644 --- a/packages/beacon-node/src/chain/archiveStore/interface.ts +++ b/packages/beacon-node/src/chain/archiveStore/interface.ts @@ -1,4 +1,4 @@ -import {CheckpointWithHex} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import {RootHex} from "@lodestar/types"; import {Metrics} from "../../metrics/metrics.js"; @@ -44,9 +44,9 @@ export type FinalizedStats = { export interface StateArchiveStrategy { onCheckpoint(stateRoot: RootHex, metrics?: Metrics | null): Promise; - onFinalizedCheckpoint(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise; - maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise; - archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise; + onFinalizedCheckpoint(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise; + maybeArchiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise; + archiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise; } export interface IArchiveStore { diff --git a/packages/beacon-node/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts b/packages/beacon-node/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts index 3e3d207a40db..09be1942c175 100644 --- a/packages/beacon-node/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts +++ b/packages/beacon-node/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts @@ -1,4 +1,4 @@ -import {CheckpointWithHex} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Epoch, RootHex, Slot} from "@lodestar/types"; @@ -9,6 +9,7 @@ import {AllocSource, BufferPool} from "../../../util/bufferPool.js"; import {getStateSlotFromBytes} from "../../../util/multifork.js"; import {IStateRegenerator} from "../../regen/interface.js"; import {serializeState} from "../../serializeState.js"; +import {fcCheckpointToHexPayload} from "../../stateCache/persistentCheckpointsCache.js"; import {StateArchiveStrategy, StatesArchiveOpts} from "../interface.js"; /** @@ -40,7 +41,7 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy { private readonly bufferPool?: BufferPool | null ) {} - async onFinalizedCheckpoint(_finalized: CheckpointWithHex, _metrics?: Metrics | null): Promise {} + async onFinalizedCheckpoint(_finalized: CheckpointWithPayloadStatus, _metrics?: Metrics | null): Promise {} async onCheckpoint(_stateRoot: RootHex, _metrics?: Metrics | null): Promise {} /** @@ -55,7 +56,7 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy { * epoch - 1024*2 epoch - 1024 epoch - 32 epoch * ``` */ - async maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise { + async maybeArchiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise { let timer = metrics?.processFinalizedCheckpoint.frequencyStateArchive.startTimer(); const lastStoredSlot = await this.db.stateArchive.lastKey(); timer?.({step: FrequencyStateArchiveStep.LoadLastStoredSlot}); @@ -104,10 +105,12 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy { * Archives finalized states from active bucket to archive bucket. * Only the new finalized state is stored to disk */ - async archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise { + async archiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise { // starting from Mar 2024, the finalized state could be from disk or in memory let timer = metrics?.processFinalizedCheckpoint.frequencyStateArchive.startTimer(); - const finalizedStateOrBytes = await this.regen.getCheckpointStateOrBytes(finalized); + // Convert fork-choice checkpoint to beacon-node checkpoint with payloadPresent + const finalizedHexPayload = fcCheckpointToHexPayload(finalized); + const finalizedStateOrBytes = await this.regen.getCheckpointStateOrBytes(finalizedHexPayload); timer?.({step: FrequencyStateArchiveStep.GetFinalizedState}); const {rootHex} = finalized; diff --git a/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts b/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts index f8f62a244aff..61c9aada1fdc 100644 --- a/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts +++ b/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts @@ -1,7 +1,7 @@ import path from "node:path"; import {ChainForkConfig} from "@lodestar/config"; import {KeyValue} from "@lodestar/db"; -import {CheckpointWithPayloadStatus, IForkChoice} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Epoch, Slot} from "@lodestar/types"; @@ -66,6 +66,7 @@ export async function archiveBlocks( // NOTE: The finalized block will be exactly the first block of `epoch` or previous const finalizedPostDeneb = finalizedCheckpoint.epoch >= config.DENEB_FORK_EPOCH; const finalizedPostFulu = finalizedCheckpoint.epoch >= config.FULU_FORK_EPOCH; + const finalizedPostGloas = finalizedCheckpoint.epoch >= config.GLOAS_FORK_EPOCH; const finalizedCanonicalBlockRoots: BlockRootSlot[] = finalizedCanonicalBlocks.map((block) => ({ slot: block.slot, @@ -103,6 +104,16 @@ export async function archiveBlocks( ); logger.verbose("Migrated dataColumnSidecars from hot DB to cold DB", {...logCtx, migratedEntries}); } + + if (finalizedPostGloas) { + const migratedEntries = await migrateExecutionPayloadEnvelopesFromHotToColdDb( + config, + db, + logger, + finalizedCanonicalBlocks + ); + logger.verbose("Migrated executionPayloadEnvelopes from hot DB to cold DB", {...logCtx, migratedEntries}); + } } // deleteNonCanonicalBlocks @@ -144,6 +155,11 @@ export async function archiveBlocks( await db.dataColumnSidecar.deleteMany(nonCanonicalBlockRoots); logger.verbose("Deleted non canonical dataColumnSidecars from hot DB", logCtx); } + + if (finalizedPostGloas) { + await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots); + logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", logCtx); + } } // Delete expired blobs @@ -372,6 +388,48 @@ async function migrateDataColumnSidecarsFromHotToColdDb( return migratedWrappedDataColumns; } +async function migrateExecutionPayloadEnvelopesFromHotToColdDb( + config: ChainForkConfig, + db: IBeaconDb, + logger: Logger, + canonicalBlocks: ProtoBlock[] +): Promise { + let migratedEnvelopes = 0; + + const payloadBlocks = canonicalBlocks.filter( + (block) => config.getForkSeq(block.slot) >= ForkSeq.gloas && block.payloadStatus === PayloadStatus.FULL + ); + if (payloadBlocks.length === 0) return 0; + const blocks = payloadBlocks.map((block) => ({slot: block.slot, root: fromHex(block.blockRoot)})); + + const envelopeEntries: KeyValue[] = []; + const migratedRoots: Uint8Array[] = []; + + const envelopeBytesArray = await Promise.all( + blocks.map((block) => db.executionPayloadEnvelope.getBinary(block.root)) + ); + + for (let i = 0; i < blocks.length; i++) { + const bytes = envelopeBytesArray[i]; + if (bytes !== null) { + envelopeEntries.push({key: blocks[i].slot, value: bytes}); + migratedRoots.push(blocks[i].root); + } else { + logger.debug("Payload in forkchoice but missing in db", {slot: blocks[i].slot, root: toRootHex(blocks[i].root)}); + } + } + + if (envelopeEntries.length > 0) { + await Promise.all([ + db.executionPayloadEnvelopeArchive.batchPutBinary(envelopeEntries), + db.executionPayloadEnvelope.batchDelete(migratedRoots), + ]); + migratedEnvelopes = envelopeEntries.length; + } + + return migratedEnvelopes; +} + /** * ``` * class SignedBeaconBlock(Container): diff --git a/packages/beacon-node/src/chain/archiveStore/utils/archivePayloads.ts b/packages/beacon-node/src/chain/archiveStore/utils/archivePayloads.ts deleted file mode 100644 index 491ae8b74b8d..000000000000 --- a/packages/beacon-node/src/chain/archiveStore/utils/archivePayloads.ts +++ /dev/null @@ -1,15 +0,0 @@ -import {CheckpointWithHex} from "@lodestar/fork-choice"; -import {IBeaconChain} from "../../interface.js"; - -/** - * Archives execution payload envelopes from hot DB to archive DB after finalization. - */ -export async function archiveExecutionPayloadEnvelopes( - chain: IBeaconChain, - _finalized: CheckpointWithHex -): Promise { - const finalizedBlock = chain.forkChoice.getFinalizedBlock(); - if (!finalizedBlock) return; - - // TODO GLOAS: Implement payload envelope archival after epbs fork choice changes are merged -} diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 0d5af688580c..c8df51f91f3a 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -7,6 +7,7 @@ import { ForkChoiceErrorCode, NotReorgedReason, getSafeExecutionBlockHash, + isGloasBlock, } from "@lodestar/fork-choice"; import {ForkPostAltair, ForkPostElectra, ForkSeq, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params"; import { @@ -30,7 +31,7 @@ import type {BeaconChain} from "../chain.js"; import {ChainEvent, ReorgEventData} from "../emitter.js"; import {ForkchoiceCaller} from "../forkChoice/index.js"; import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "../reprocess.js"; -import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js"; +import {toCheckpointHexPayload} from "../stateCache/persistentCheckpointsCache.js"; import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js"; import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js"; import {getCheckpointFromState} from "./utils/checkpoint.js"; @@ -116,7 +117,11 @@ export async function importBlock( // This adds the state necessary to process the next block // Some block event handlers require state being in state cache so need to do this before emitting EventType.block - this.regen.processState(blockRootHex, postState); + // Pre-Gloas: blockSummary.payloadStatus is always FULL, payloadPresent = true + // Post-Gloas: blockSummary.payloadStatus is always PENDING, so payloadPresent = false (block state only, no payload processing yet) + const payloadPresent = !isGloasBlock(blockSummary); + // processState manages both block state and payload state variants together for memory/disk management + this.regen.processBlockState(blockRootHex, postState); this.metrics?.importBlock.bySource.inc({source: source.source}); this.logger.verbose("Added block to forkchoice and state cache", {slot: blockSlot, root: blockRootHex}); @@ -456,12 +461,12 @@ export async function importBlock( // Cache state to preserve epoch transition work const checkpointState = postState; const cp = getCheckpointFromState(checkpointState); - this.regen.addCheckpointState(cp, checkpointState); + this.regen.addCheckpointState(cp, checkpointState, payloadPresent); // consumers should not mutate state ever this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState); // Note: in-lined code from previos handler of ChainEvent.checkpoint - this.logger.verbose("Checkpoint processed", toCheckpointHex(cp)); + this.logger.verbose("Checkpoint processed", toCheckpointHexPayload(cp, payloadPresent)); const activeValidatorsCount = checkpointState.epochCtx.currentShuffling.activeIndices.length; this.metrics?.currentActiveValidators.set(activeValidatorsCount); @@ -479,7 +484,7 @@ export async function importBlock( const justifiedEpoch = justifiedCheckpoint.epoch; const preJustifiedEpoch = parentBlockSummary.justifiedEpoch; if (justifiedEpoch > preJustifiedEpoch) { - this.logger.verbose("Checkpoint justified", toCheckpointHex(justifiedCheckpoint)); + this.logger.verbose("Checkpoint justified", toCheckpointHexPayload(justifiedCheckpoint, payloadPresent)); this.metrics?.previousJustifiedEpoch.set(checkpointState.previousJustifiedCheckpoint.epoch); this.metrics?.currentJustifiedEpoch.set(justifiedCheckpoint.epoch); } @@ -493,7 +498,7 @@ export async function importBlock( state: toRootHex(checkpointState.hashTreeRoot()), executionOptimistic: false, }); - this.logger.verbose("Checkpoint finalized", toCheckpointHex(finalizedCheckpoint)); + this.logger.verbose("Checkpoint finalized", toCheckpointHexPayload(finalizedCheckpoint, payloadPresent)); this.metrics?.finalizedEpoch.set(finalizedCheckpoint.epoch); } } diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index f0a9aeace741..4c41afd33aa6 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -3,11 +3,12 @@ import {PrivateKey} from "@libp2p/interface"; import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; import { - CheckpointWithHex, CheckpointWithPayloadStatus, IForkChoice, + PayloadStatus, ProtoBlock, UpdateHeadOpt, + getCheckpointPayloadStatus, } from "@lodestar/fork-choice"; import {LoggerNode} from "@lodestar/logger/node"; import { @@ -126,7 +127,7 @@ import {DbCPStateDatastore, checkpointToDatastoreKey} from "./stateCache/datasto import {FileCPStateDatastore} from "./stateCache/datastore/file.js"; import {CPStateDatastore} from "./stateCache/datastore/types.js"; import {FIFOBlockStateCache} from "./stateCache/fifoBlockStateCache.js"; -import {PersistentCheckpointStateCache} from "./stateCache/persistentCheckpointsCache.js"; +import {PersistentCheckpointStateCache, fcCheckpointToHexPayload} from "./stateCache/persistentCheckpointsCache.js"; import {CheckpointStateCache} from "./stateCache/types.js"; import {ValidatorMonitor} from "./validatorMonitor.js"; @@ -375,7 +376,8 @@ export class BeaconChain implements IBeaconChain { const {checkpoint} = computeAnchorCheckpoint(config, anchorState); blockStateCache.add(anchorState); blockStateCache.setHeadState(anchorState); - checkpointStateCache.add(checkpoint, anchorState); + const payloadPresent = getCheckpointPayloadStatus(anchorState, checkpoint.epoch) === PayloadStatus.FULL; + checkpointStateCache.add(checkpoint, anchorState, payloadPresent); const forkChoice = initializeForkChoice( config, @@ -648,15 +650,18 @@ export class BeaconChain implements IBeaconChain { return this.cpStateDatastore.readLatestSafe(); } - const persistedKey = checkpointToDatastoreKey(checkpoint); + // TODO GLOAS: Need to revisit the design of this api. Currently we just retrieve FULL state of the checkpoint for backwards compatibility. + // because pre-gloas we always store FULL checkpoint state. + const persistedKey = checkpointToDatastoreKey(checkpoint, true); return this.cpStateDatastore.read(persistedKey); } getStateByCheckpoint( - checkpoint: CheckpointWithHex + checkpoint: CheckpointWithPayloadStatus ): {state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null { // finalized or justified checkpoint states maynot be available with PersistentCheckpointStateCache, use getCheckpointStateOrBytes() api to get Uint8Array - const cachedStateCtx = this.regen.getCheckpointStateSync(checkpoint); + const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint); + const cachedStateCtx = this.regen.getCheckpointStateSync(checkpointHexPayload); if (cachedStateCtx) { const block = this.forkChoice.getBlockDefaultStatus(cachedStateCtx.latestBlockHeader.hashTreeRoot()); const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; @@ -671,9 +676,10 @@ export class BeaconChain implements IBeaconChain { } async getStateOrBytesByCheckpoint( - checkpoint: CheckpointWithHex + checkpoint: CheckpointWithPayloadStatus ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null> { - const cachedStateCtx = await this.regen.getCheckpointStateOrBytes(checkpoint); + const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint); + const cachedStateCtx = await this.regen.getCheckpointStateOrBytes(checkpointHexPayload); if (cachedStateCtx) { const block = this.forkChoice.getBlockDefaultStatus(checkpoint.root); const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; @@ -1236,7 +1242,8 @@ export class BeaconChain implements IBeaconChain { checkpoint: CheckpointWithPayloadStatus, blockState: CachedBeaconStateAllForks ): {state: CachedBeaconStateAllForks; stateId: string; shouldWarn: boolean} { - const state = this.regen.getCheckpointStateSync(checkpoint); + const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint); + const state = this.regen.getCheckpointStateSync(checkpointHexPayload); if (state) { return {state, stateId: "checkpoint_state", shouldWarn: false}; } @@ -1363,6 +1370,10 @@ export class BeaconChain implements IBeaconChain { private onClockEpoch(epoch: Epoch): void { this.metrics?.clockEpoch.set(epoch); + if (epoch === this.config.GLOAS_FORK_EPOCH) { + this.regen.upgradeForGloas(epoch); + } + this.seenAttesters.prune(epoch); this.seenAggregators.prune(epoch); this.seenPayloadAttesters.prune(epoch); @@ -1376,7 +1387,7 @@ export class BeaconChain implements IBeaconChain { this.seenContributionAndProof.prune(head.slot); } - private onForkChoiceJustified(this: BeaconChain, cp: CheckpointWithHex): void { + private onForkChoiceJustified(this: BeaconChain, cp: CheckpointWithPayloadStatus): void { this.logger.verbose("Fork choice justified", {epoch: cp.epoch, root: cp.rootHex}); } @@ -1387,7 +1398,7 @@ export class BeaconChain implements IBeaconChain { }); } - private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithHex): Promise { + private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithPayloadStatus): Promise { this.logger.verbose("Fork choice finalized", {epoch: cp.epoch, root: cp.rootHex}); const finalizedSlot = computeStartSlotAtEpoch(cp.epoch); this.seenBlockProposers.prune(finalizedSlot); @@ -1429,7 +1440,7 @@ export class BeaconChain implements IBeaconChain { } } - private async updateValidatorsCustodyRequirement(finalizedCheckpoint: CheckpointWithHex): Promise { + private async updateValidatorsCustodyRequirement(finalizedCheckpoint: CheckpointWithPayloadStatus): Promise { if (this.custodyConfig.targetCustodyGroupCount === this.config.NUMBER_OF_CUSTODY_GROUPS) { // Custody requirements can only be increased, we can disable dynamic custody updates // if the node already maintains custody of all custody groups in case it is configured diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index cc300e9a5dd7..d15c9951bfbb 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -1,6 +1,6 @@ import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; -import {CheckpointWithHex, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {CheckpointWithHex, CheckpointWithPayloadStatus, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {BeaconStateAllForks, CachedBeaconStateAllForks, EpochShuffling, PubkeyCache} from "@lodestar/state-transition"; import { BeaconBlock, @@ -192,7 +192,7 @@ export interface IBeaconChain { ): {state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null; /** Return state bytes by checkpoint */ getStateOrBytesByCheckpoint( - checkpoint: CheckpointWithHex + checkpoint: CheckpointWithPayloadStatus ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null>; /** diff --git a/packages/beacon-node/src/chain/prepareNextSlot.ts b/packages/beacon-node/src/chain/prepareNextSlot.ts index 0d84adc771fd..0f4b0d0f6191 100644 --- a/packages/beacon-node/src/chain/prepareNextSlot.ts +++ b/packages/beacon-node/src/chain/prepareNextSlot.ts @@ -1,6 +1,6 @@ import {routes} from "@lodestar/api"; import {ChainForkConfig} from "@lodestar/config"; -import {getSafeExecutionBlockHash} from "@lodestar/fork-choice"; +import {PayloadStatus, getSafeExecutionBlockHash} from "@lodestar/fork-choice"; import {ForkPostBellatrix, ForkSeq, SLOTS_PER_EPOCH, isForkPostBellatrix} from "@lodestar/params"; import { CachedBeaconStateAllForks, @@ -211,7 +211,11 @@ export class PrepareNextSlotScheduler { // + if next slot is a skipped slot, it'd help getting target checkpoint state faster to validate attestations if (isEpochTransition) { this.metrics?.precomputeNextEpochTransition.count.inc({result: "success"}, 1); - const previousHits = this.chain.regen.updatePreComputedCheckpoint(headRoot, nextEpoch); + // Determine payloadPresent from head block's payload status + // Pre-Gloas: payloadStatus is always FULL → payloadPresent = true + // Post-Gloas: FULL → true, EMPTY → false, PENDING → false (conservative, treat as block state) + const payloadPresent = headBlock.payloadStatus === PayloadStatus.FULL; + const previousHits = this.chain.regen.updatePreComputedCheckpoint(headRoot, nextEpoch, payloadPresent); if (previousHits === 0) { this.metrics?.precomputeNextEpochTransition.waste.inc(); } diff --git a/packages/beacon-node/src/chain/regen/errors.ts b/packages/beacon-node/src/chain/regen/errors.ts index eb41e8321da3..6352d765ea11 100644 --- a/packages/beacon-node/src/chain/regen/errors.ts +++ b/packages/beacon-node/src/chain/regen/errors.ts @@ -1,3 +1,4 @@ +import {PayloadStatus} from "@lodestar/fork-choice"; import {Root, RootHex, Slot} from "@lodestar/types"; export enum RegenErrorCode { @@ -9,6 +10,8 @@ export enum RegenErrorCode { BLOCK_NOT_IN_DB = "REGEN_ERROR_BLOCK_NOT_IN_DB", STATE_TRANSITION_ERROR = "REGEN_ERROR_STATE_TRANSITION_ERROR", INVALID_STATE_ROOT = "REGEN_ERROR_INVALID_STATE_ROOT", + UNEXPECTED_PAYLOAD_STATUS = "REGEN_ERROR_UNEXPECTED_PAYLOAD_STATUS", + INTERNAL_ERROR = "REGEN_ERROR_INTERNAL_ERROR", } export type RegenErrorType = @@ -19,7 +22,9 @@ export type RegenErrorType = | {code: RegenErrorCode.TOO_MANY_BLOCK_PROCESSED; stateRoot: RootHex | Root} | {code: RegenErrorCode.BLOCK_NOT_IN_DB; blockRoot: RootHex | Root} | {code: RegenErrorCode.STATE_TRANSITION_ERROR; error: Error} - | {code: RegenErrorCode.INVALID_STATE_ROOT; slot: Slot; expected: RootHex; actual: RootHex}; + | {code: RegenErrorCode.INVALID_STATE_ROOT; slot: Slot; expected: RootHex; actual: RootHex} + | {code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS; blockRoot: RootHex | Root; payloadStatus: PayloadStatus} + | {code: RegenErrorCode.INTERNAL_ERROR; message: string}; export class RegenError extends Error { type: RegenErrorType; diff --git a/packages/beacon-node/src/chain/regen/interface.ts b/packages/beacon-node/src/chain/regen/interface.ts index 61b68fa55625..019523a8139f 100644 --- a/packages/beacon-node/src/chain/regen/interface.ts +++ b/packages/beacon-node/src/chain/regen/interface.ts @@ -2,7 +2,7 @@ import {routes} from "@lodestar/api"; import {ProtoBlock} from "@lodestar/fork-choice"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {BeaconBlock, Epoch, RootHex, Slot, phase0} from "@lodestar/types"; -import {CheckpointHex} from "../stateCache/types.js"; +import {CheckpointHexPayload} from "../stateCache/types.js"; export enum RegenCaller { getDuties = "getDuties", @@ -38,15 +38,21 @@ export interface IStateRegenerator extends IStateRegeneratorInternal { dumpCacheSummary(): routes.lodestar.StateCacheItem[]; getStateSync(stateRoot: RootHex): CachedBeaconStateAllForks | null; getPreStateSync(block: BeaconBlock): CachedBeaconStateAllForks | null; - getCheckpointStateOrBytes(cp: CheckpointHex): Promise; - getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null; + getCheckpointStateOrBytes(cp: CheckpointHexPayload): Promise; + getCheckpointStateSync(cp: CheckpointHexPayload): CachedBeaconStateAllForks | null; getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null; pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void; pruneOnFinalized(finalizedEpoch: Epoch): void; - processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void; - addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void; + processBlockState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void; + processPayloadState(payloadState: CachedBeaconStateAllForks): void; + /** + * payloadPresent is true if this is payload state, false if block state. + * payloadPresent is always true for pre-gloas. + */ + addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks, payloadPresent: boolean): void; updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void; - updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null; + updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null; + upgradeForGloas(epoch: Epoch): void; } /** diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index 04011d53cbd1..0da06877faa8 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -1,11 +1,11 @@ import {routes} from "@lodestar/api"; -import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition"; import {BeaconBlock, Epoch, RootHex, Slot, isGloasBeaconBlock, phase0} from "@lodestar/types"; -import {Logger, toRootHex} from "@lodestar/utils"; +import {Logger, fromHex, toRootHex} from "@lodestar/utils"; import {Metrics} from "../../metrics/index.js"; import {JobItemQueue} from "../../util/queue/index.js"; -import {BlockStateCache, CheckpointHex, CheckpointStateCache} from "../stateCache/types.js"; +import {BlockStateCache, CheckpointHexPayload, CheckpointStateCache} from "../stateCache/types.js"; import {RegenError, RegenErrorCode} from "./errors.js"; import { IStateRegenerator, @@ -104,9 +104,19 @@ export class QueuedStateRegenerator implements IStateRegenerator { const parentEpoch = computeEpochAtSlot(parentBlock.slot); const blockEpoch = computeEpochAtSlot(block.slot); + // Convert PayloadStatus to payloadPresent boolean + if (parentBlock.payloadStatus === PayloadStatus.PENDING) { + throw new RegenError({ + code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS, + blockRoot: block.parentRoot, + payloadStatus: parentBlock.payloadStatus, + }); + } + const payloadPresent = parentBlock.payloadStatus === PayloadStatus.FULL; + // Check the checkpoint cache (if the pre-state is a checkpoint state) if (parentEpoch < blockEpoch) { - const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch); + const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch, payloadPresent); if (checkpointState && computeEpochAtSlot(checkpointState.slot) === blockEpoch) { return checkpointState; } @@ -125,14 +135,14 @@ export class QueuedStateRegenerator implements IStateRegenerator { return null; } - async getCheckpointStateOrBytes(cp: CheckpointHex): Promise { + async getCheckpointStateOrBytes(cp: CheckpointHexPayload): Promise { return this.checkpointStateCache.getStateOrBytes(cp); } /** * Get checkpoint state from cache */ - getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null { + getCheckpointStateSync(cp: CheckpointHexPayload): CachedBeaconStateAllForks | null { return this.checkpointStateCache.get(cp); } @@ -140,7 +150,19 @@ export class QueuedStateRegenerator implements IStateRegenerator { * Get state closest to head */ getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null { - return this.checkpointStateCache.getLatest(head.blockRoot, Infinity) || this.blockStateCache.get(head.stateRoot); + // Convert PayloadStatus to payloadPresent boolean + if (head.payloadStatus === PayloadStatus.PENDING) { + throw new RegenError({ + code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS, + blockRoot: fromHex(head.blockRoot), + payloadStatus: head.payloadStatus, + }); + } + const payloadPresent = head.payloadStatus === PayloadStatus.FULL; + return ( + this.checkpointStateCache.getLatest(head.blockRoot, Infinity, payloadPresent) || + this.blockStateCache.get(head.stateRoot) + ); } pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void { @@ -153,15 +175,24 @@ export class QueuedStateRegenerator implements IStateRegenerator { this.blockStateCache.deleteAllBeforeEpoch(finalizedEpoch); } - processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void { + processBlockState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void { this.blockStateCache.add(postState); this.checkpointStateCache.processState(blockRootHex, postState).catch((e) => { this.logger.debug("Error processing block state", {blockRootHex, slot: postState.slot}, e); }); } - addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void { - this.checkpointStateCache.add(cp, item); + /** + * Process payload state for caching after importing execution payload. + */ + processPayloadState(payloadState: CachedBeaconStateAllForks): void { + // Add payload state to block state cache (keyed by payload state root) + this.blockStateCache.add(payloadState); + } + + // TODO GLOAS: This should also be called when importing execution payload after we implement it + addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks, payloadPresent: boolean): void { + this.checkpointStateCache.add(cp, item, payloadPresent); } updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void { @@ -197,8 +228,13 @@ export class QueuedStateRegenerator implements IStateRegenerator { } } - updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null { - return this.checkpointStateCache.updatePreComputedCheckpoint(rootHex, epoch); + updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null { + return this.checkpointStateCache.updatePreComputedCheckpoint(rootHex, epoch, payloadPresent); + } + + upgradeForGloas(epoch: Epoch): void { + this.logger.verbose("Upgrading block state cache for Gloas fork", {epoch}); + this.blockStateCache.upgradeToGloas(); } /** diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index 08e580f1295c..8a00364a5f50 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -1,6 +1,6 @@ import {ChainForkConfig} from "@lodestar/config"; -import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; -import {SLOTS_PER_EPOCH} from "@lodestar/params"; +import {IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; +import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; import { CachedBeaconStateAllForks, DataAvailabilityStatus, @@ -111,9 +111,20 @@ export class StateRegenerator implements IStateRegeneratorInternal { const {blockRoot} = block; const {checkpointStateCache} = this.modules; const epoch = computeEpochAtSlot(slot); + + // Convert PayloadStatus to payloadPresent boolean + if (block.payloadStatus === PayloadStatus.PENDING) { + throw new RegenError({ + code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS, + blockRoot: fromHex(blockRoot), + payloadStatus: block.payloadStatus, + }); + } + const payloadPresent = block.payloadStatus === PayloadStatus.FULL; + const latestCheckpointStateCtx = allowDiskReload - ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch) - : checkpointStateCache.getLatest(blockRoot, epoch); + ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch, payloadPresent) + : checkpointStateCache.getLatest(blockRoot, epoch, payloadPresent); // If a checkpoint state exists with the given checkpoint root, it either is in requested epoch // or needs to have empty slots processed until the requested epoch @@ -166,9 +177,19 @@ export class StateRegenerator implements IStateRegeneratorInternal { const lastBlockToReplay = blocksToReplay.at(-1); if (!lastBlockToReplay) continue; const epoch = computeEpochAtSlot(lastBlockToReplay.slot - 1); + + // Convert PayloadStatus to payloadPresent boolean + if (b.payloadStatus === PayloadStatus.PENDING) { + throw new RegenError({ + code: RegenErrorCode.INTERNAL_ERROR, + message: `Unexpected PENDING payloadStatus for ancestor block ${b.blockRoot} at slot ${b.slot}`, + }); + } + const payloadPresent = b.payloadStatus === PayloadStatus.FULL; + state = allowDiskReload - ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch) - : checkpointStateCache.getLatest(b.blockRoot, epoch); + ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch, payloadPresent) + : checkpointStateCache.getLatest(b.blockRoot, epoch, payloadPresent); if (state) { break; } @@ -332,6 +353,11 @@ async function processSlotsByCheckpoint( * emitting "checkpoint" events after every epoch processed. * * Stops processing after no more full epochs can be processed. + * + * Output state variant: + * - Post-Gloas: If slots are processed, returns block state (payloadPresent=false). + * If no slots processed, returns preState as-is (preserves variant). + * - Pre-Gloas: Always payloadPresent=true (no block/payload distinction). */ export async function processSlotsToNearestCheckpoint( modules: { @@ -374,7 +400,11 @@ export async function processSlotsToNearestCheckpoint( // This may becomes the "official" checkpoint state if the 1st block of epoch is skipped const checkpointState = postState; const cp = getCheckpointFromState(checkpointState); - checkpointStateCache.add(cp, checkpointState); + // processSlots() only does epoch transitions, never processes payloads + // Pre-Gloas: payloadPresent is always true (execution payload embedded in block) + // Post-Gloas: result is a block state (payloadPresent=false) + const isPayloadPresent = checkpointState.config.getForkSeq(checkpointState.slot) < ForkSeq.gloas; + checkpointStateCache.add(cp, checkpointState, isPayloadPresent); // consumers should not mutate state ever emitter?.emit(ChainEvent.checkpoint, cp, checkpointState); diff --git a/packages/beacon-node/src/chain/stateCache/datastore/db.ts b/packages/beacon-node/src/chain/stateCache/datastore/db.ts index 64c893c9bdd7..7d49169960d9 100644 --- a/packages/beacon-node/src/chain/stateCache/datastore/db.ts +++ b/packages/beacon-node/src/chain/stateCache/datastore/db.ts @@ -1,6 +1,6 @@ import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {Epoch, phase0, ssz} from "@lodestar/types"; -import {MapDef} from "@lodestar/utils"; +import {MapDef, byteArrayEquals} from "@lodestar/utils"; import {IBeaconDb} from "../../../db/interface.js"; import { getLastProcessedSlotFromBeaconStateSerialized, @@ -14,8 +14,8 @@ import {CPStateDatastore, DatastoreKey} from "./types.js"; export class DbCPStateDatastore implements CPStateDatastore { constructor(private readonly db: IBeaconDb) {} - async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array): Promise { - const serializedCheckpoint = checkpointToDatastoreKey(cpKey); + async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean): Promise { + const serializedCheckpoint = checkpointToDatastoreKey(cpKey, payloadPresent); await this.db.checkpointState.putBinary(serializedCheckpoint, stateBytes); return serializedCheckpoint; } @@ -40,18 +40,30 @@ export class DbCPStateDatastore implements CPStateDatastore { } } +function extractCheckpointBytes(key: DatastoreKey): Uint8Array { + const fixedSize = ssz.phase0.Checkpoint.minSize; + return key.subarray(0, fixedSize); +} + export function datastoreKeyToCheckpoint(key: DatastoreKey): phase0.Checkpoint { - return ssz.phase0.Checkpoint.deserialize(key); + return ssz.phase0.Checkpoint.deserialize(extractCheckpointBytes(key)); +} + +export function checkpointToDatastoreKey(cp: phase0.Checkpoint, payloadPresent: boolean): DatastoreKey { + const cpBytes = ssz.phase0.Checkpoint.serialize(cp); + const key = new Uint8Array(cpBytes.length + 1); + key.set(cpBytes); + key[cpBytes.length] = payloadPresent ? 1 : 0; + return key; } -export function checkpointToDatastoreKey(cp: phase0.Checkpoint): DatastoreKey { - return ssz.phase0.Checkpoint.serialize(cp); +function isPayloadCheckpointState(key: DatastoreKey): boolean { + return key.at(-1) === 1; } /** - * Get the latest safe checkpoint state the node can use to boot from - * - it should be the checkpoint state that's unique in its epoch - * - its last processed block slot should be at epoch boundary or last slot of previous epoch + * Get the latest "safe" checkpoint state the node can use to boot from + * - its last processed block slot should be at epoch boundary (CRCS) or last slot of previous epoch (PRCS) * - state slot should be at epoch boundary * - state slot should be equal to epoch * SLOTS_PER_EPOCH * @@ -70,9 +82,20 @@ export async function getLatestSafeDatastoreKey( const dataStoreKeyByEpoch: Map = new Map(); for (const [epoch, keys] of checkpointsByEpoch.entries()) { - // only consider epochs with a single checkpoint to avoid ambiguity from forks if (keys.length === 1) { + // PRCS (skipped slot) or CRCS and no payloadPresent + // Pre-gloas always fall into this case dataStoreKeyByEpoch.set(epoch, keys[0]); + } else if (keys.length === 2) { + // CRCS without payload and CRCS with payload + // ie Two keys for the same checkpoint with different payloadPresent suffix (FULL/EMPTY) + // TODO GLOAS: Here we pick FULL key, there is a chance that payload is orphaned hence we not be able to sync + const cp0 = extractCheckpointBytes(keys[0]); + const cp1 = extractCheckpointBytes(keys[1]); + if (byteArrayEquals(cp0, cp1)) { + const fullKey = isPayloadCheckpointState(keys[0]) ? keys[0] : keys[1]; + dataStoreKeyByEpoch.set(epoch, fullKey); + } } } diff --git a/packages/beacon-node/src/chain/stateCache/datastore/file.ts b/packages/beacon-node/src/chain/stateCache/datastore/file.ts index 15ccbfb81f43..6c0391c3c29a 100644 --- a/packages/beacon-node/src/chain/stateCache/datastore/file.ts +++ b/packages/beacon-node/src/chain/stateCache/datastore/file.ts @@ -1,12 +1,13 @@ import path from "node:path"; -import {phase0, ssz} from "@lodestar/types"; +import {phase0} from "@lodestar/types"; import {fromHex, toHex} from "@lodestar/utils"; import {ensureDir, readFile, readFileNames, removeFile, writeIfNotExist} from "../../../util/file.js"; -import {getLatestSafeDatastoreKey} from "./db.js"; +import {checkpointToDatastoreKey, getLatestSafeDatastoreKey} from "./db.js"; import {CPStateDatastore, DatastoreKey} from "./types.js"; const CHECKPOINT_STATES_FOLDER = "checkpoint_states"; -const CHECKPOINT_FILE_NAME_LENGTH = 82; +/** 41 bytes (40 checkpoint + 1 payloadPresent) = 82 hex chars + "0x" prefix = 84 */ +const CHECKPOINT_FILE_NAME_LENGTH = 84; /** * Implementation of CPStateDatastore using file system, this is beneficial for debugging. @@ -28,8 +29,8 @@ export class FileCPStateDatastore implements CPStateDatastore { } } - async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array): Promise { - const serializedCheckpoint = ssz.phase0.Checkpoint.serialize(cpKey); + async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean): Promise { + const serializedCheckpoint = checkpointToDatastoreKey(cpKey, payloadPresent); const filePath = path.join(this.folderPath, toHex(serializedCheckpoint)); await writeIfNotExist(filePath, stateBytes); return serializedCheckpoint; diff --git a/packages/beacon-node/src/chain/stateCache/datastore/types.ts b/packages/beacon-node/src/chain/stateCache/datastore/types.ts index c63c54cca1d1..de25274539d9 100644 --- a/packages/beacon-node/src/chain/stateCache/datastore/types.ts +++ b/packages/beacon-node/src/chain/stateCache/datastore/types.ts @@ -1,11 +1,12 @@ import {phase0} from "@lodestar/types"; -// With db implementation, persistedKey is serialized data of a checkpoint +// With db implementation, persistedKey is serialized data of a checkpoint + 1 +// ie a fixed size of `ssz.phase0.Checkpoint.minSize + 1` export type DatastoreKey = Uint8Array; // Make this generic to support testing export interface CPStateDatastore { - write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array) => Promise; + write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean) => Promise; remove: (key: DatastoreKey) => Promise; read: (key: DatastoreKey) => Promise; readLatestSafe: () => Promise; diff --git a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts index 37af369e28fd..be3d9b2d3510 100644 --- a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts +++ b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts @@ -20,6 +20,11 @@ export type FIFOBlockStateCacheOpts = { * clock slot */ export const DEFAULT_MAX_BLOCK_STATES = 64; +/** + * For Gloas (ePBS), each block can have two states: block state and payload state. + * Double the cache size to maintain the same effective block depth. + */ +export const DEFAULT_MAX_BLOCK_STATES_GLOAS = 128; /** * New implementation of BlockStateCache that keeps the most recent n states consistently @@ -41,10 +46,7 @@ export const DEFAULT_MAX_BLOCK_STATES = 64; * The maintained key order would be: 11 -> 13 -> 12 -> 10, and state 10 will be pruned first. */ export class FIFOBlockStateCache implements BlockStateCache { - /** - * Max number of states allowed in the cache - */ - readonly maxStates: number; + private maxStates: number; private readonly cache: MapTracker; /** @@ -170,6 +172,10 @@ export class FIFOBlockStateCache implements BlockStateCache { } } + upgradeToGloas(): void { + this.maxStates = DEFAULT_MAX_BLOCK_STATES_GLOAS; + } + /** * No need for this implementation * This is only to conform to the old api diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 862b063711f3..4562bac1a52b 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -1,5 +1,6 @@ import {routes} from "@lodestar/api"; import {BeaconConfig} from "@lodestar/config"; +import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import { CachedBeaconStateAllForks, computeStartSlotAtEpoch, @@ -14,7 +15,7 @@ import {IClock} from "../../util/clock.js"; import {serializeState} from "../serializeState.js"; import {CPStateDatastore, DatastoreKey} from "./datastore/index.js"; import {MapTracker} from "./mapMetrics.js"; -import {BlockStateCache, CacheItemType, CheckpointHex, CheckpointStateCache} from "./types.js"; +import {BlockStateCache, CacheItemType, CheckpointHexPayload, CheckpointStateCache} from "./types.js"; export type PersistentCheckpointStateCacheOpts = { /** Keep max n state epochs in memory, persist the rest to disk */ @@ -54,6 +55,22 @@ type CacheItem = InMemoryCacheItem | PersistedCacheItem; type LoadedStateBytesData = {persistedKey: DatastoreKey; stateBytes: Uint8Array}; +/** Bitmask for tracking which payload variants exist per root in the epochIndex */ +enum PayloadAvailability { + NOT_PRESENT = 1, + PRESENT = 2, +} + +const PAYLOAD_AVAILABILITY_ALL = [PayloadAvailability.NOT_PRESENT, PayloadAvailability.PRESENT] as const; + +function toPayloadAvailability(payloadPresent: boolean): PayloadAvailability { + return payloadPresent ? PayloadAvailability.PRESENT : PayloadAvailability.NOT_PRESENT; +} + +function fromPayloadAvailability(flag: PayloadAvailability): boolean { + return flag === PayloadAvailability.PRESENT; +} + /** * Before n-historical states, lodestar keeps all checkpoint states since finalized * Since Sep 2024, lodestar stores 3 most recent checkpoint states in memory and the rest on disk. The finalized state @@ -106,8 +123,8 @@ const PROCESS_CHECKPOINT_STATES_BPS = 6667; */ export class PersistentCheckpointStateCache implements CheckpointStateCache { private readonly cache: MapTracker; - /** Epoch -> Set */ - private readonly epochIndex = new MapDef>(() => new Set()); + /** Epoch -> Map */ + private readonly epochIndex = new MapDef>(() => new Map()); private readonly config: BeaconConfig; private readonly metrics: Metrics | null | undefined; private readonly logger: Logger; @@ -203,13 +220,18 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * - Get block for processing * - Regen head state */ - async getOrReload(cp: CheckpointHex): Promise { + async getOrReload(cp: CheckpointHexPayload): Promise { const stateOrStateBytesData = await this.getStateOrLoadDb(cp); if (stateOrStateBytesData === null || isCachedBeaconState(stateOrStateBytesData)) { return stateOrStateBytesData ?? null; } const {persistedKey, stateBytes} = stateOrStateBytesData; - const logMeta = {persistedKey: toHex(persistedKey)}; + const logMeta = { + epoch: cp.epoch, + rootHex: cp.rootHex, + payloadPresent: cp.payloadPresent, + persistedKey: toHex(persistedKey), + }; this.logger.debug("Reload: read state successful", logMeta); this.metrics?.cpStateCache.stateReloadSecFromSlot.observe( this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0 @@ -250,7 +272,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { // only remove persisted state once we reload successfully const cpKey = toCacheKey(cp); this.cache.set(cpKey, {type: CacheItemType.inMemory, state: newCachedState, persistedKey}); - this.epochIndex.getOrDefault(cp.epoch).add(cp.rootHex); + this.addToEpochIndex(cp.epoch, cp.rootHex, cp.payloadPresent); // don't prune from memory here, call it at the last 1/3 of slot 0 of an epoch return newCachedState; } catch (e) { @@ -262,7 +284,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Return either state or state bytes loaded from db. */ - async getStateOrBytes(cp: CheckpointHex): Promise { + async getStateOrBytes(cp: CheckpointHexPayload): Promise { const stateOrLoadedState = await this.getStateOrLoadDb(cp); if (stateOrLoadedState === null || isCachedBeaconState(stateOrLoadedState)) { return stateOrLoadedState; @@ -273,7 +295,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Return either state or state bytes with persisted key loaded from db. */ - async getStateOrLoadDb(cp: CheckpointHex): Promise { + async getStateOrLoadDb(cp: CheckpointHexPayload): Promise { const cpKey = toCacheKey(cp); const inMemoryState = this.get(cpKey); if (inMemoryState) { @@ -304,7 +326,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Similar to get() api without reloading from disk */ - get(cpOrKey: CheckpointHex | string): CachedBeaconStateAllForks | null { + get(cpOrKey: CheckpointHexPayload | CacheKey): CachedBeaconStateAllForks | null { this.metrics?.cpStateCache.lookups.inc(); const cpKey = typeof cpOrKey === "string" ? cpOrKey : toCacheKey(cpOrKey); const cacheItem = this.cache.get(cpKey); @@ -330,9 +352,11 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Add a state of a checkpoint to this cache, prune from memory if necessary. + * @param payloadPresent - For Gloas: true if this is payload state, false if block state. + * Always true for pre-Gloas. */ - add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks): void { - const cpHex = toCheckpointHex(cp); + add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks, payloadPresent: boolean): void { + const cpHex = toCheckpointHexPayload(cp, payloadPresent); const key = toCacheKey(cpHex); const cacheItem = this.cache.get(key); this.metrics?.cpStateCache.adds.inc(); @@ -343,27 +367,32 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.logger.verbose("Added checkpoint state to memory but a persisted key existed", { epoch: cp.epoch, rootHex: cpHex.rootHex, + payloadPresent, persistedKey: toHex(persistedKey), }); } else { this.cache.set(key, {type: CacheItemType.inMemory, state}); - this.logger.verbose("Added checkpoint state to memory", {epoch: cp.epoch, rootHex: cpHex.rootHex}); + this.logger.verbose("Added checkpoint state to memory", { + epoch: cp.epoch, + rootHex: cpHex.rootHex, + payloadPresent, + }); } - this.epochIndex.getOrDefault(cp.epoch).add(cpHex.rootHex); + this.addToEpochIndex(cp.epoch, cpHex.rootHex, cpHex.payloadPresent); this.prunePersistedStates(); } /** * Searches in-memory state for the latest cached state with a `root` without reload, starting with `epoch` and descending */ - getLatest(rootHex: RootHex, maxEpoch: Epoch): CachedBeaconStateAllForks | null { + getLatest(rootHex: RootHex, maxEpoch: Epoch, payloadPresent: boolean): CachedBeaconStateAllForks | null { // sort epochs in descending order, only consider epochs lte `epoch` const epochs = Array.from(this.epochIndex.keys()) .sort((a, b) => b - a) .filter((e) => e <= maxEpoch); for (const epoch of epochs) { - if (this.epochIndex.get(epoch)?.has(rootHex)) { - const inMemoryClonedState = this.get({rootHex, epoch}); + if (this.hasPayloadVariant(epoch, rootHex, payloadPresent)) { + const inMemoryClonedState = this.get({rootHex, epoch, payloadPresent}); if (inMemoryClonedState) { return inMemoryClonedState; } @@ -379,20 +408,24 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * - Get block for processing * - Regen head state */ - async getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch): Promise { + async getOrReloadLatest( + rootHex: RootHex, + maxEpoch: Epoch, + payloadPresent: boolean + ): Promise { // sort epochs in descending order, only consider epochs lte `epoch` const epochs = Array.from(this.epochIndex.keys()) .sort((a, b) => b - a) .filter((e) => e <= maxEpoch); for (const epoch of epochs) { - if (this.epochIndex.get(epoch)?.has(rootHex)) { + if (this.hasPayloadVariant(epoch, rootHex, payloadPresent)) { try { - const state = await this.getOrReload({rootHex, epoch}); + const state = await this.getOrReload({rootHex, epoch, payloadPresent}); if (state) { return state; } } catch (e) { - this.logger.debug("Error get or reload state", {epoch, rootHex}, e as Error); + this.logger.debug("Error get or reload state", {epoch, rootHex, payloadPresent}, e as Error); } } } @@ -400,12 +433,14 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } /** - * Update the precomputed checkpoint and return the number of his for the + * Update the precomputed checkpoint and return the number of hits for the * previous one (if any). + * @param payloadPresent - For Gloas: true if head block has FULL payload, false if EMPTY. + * Always true for pre-Gloas. */ - updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null { + updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null { const previousHits = this.preComputedCheckpointHits; - this.preComputedCheckpoint = toCacheKey({rootHex, epoch}); + this.preComputedCheckpoint = toCacheKey({rootHex, epoch, payloadPresent}); this.preComputedCheckpointHits = 0; return previousHits; } @@ -479,6 +514,9 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * - 2 then we'll persist {root: b2, epoch n-2} checkpoint state to disk, there are also 2 checkpoint states in memory at epoch n, same to the above (maxEpochsInMemory=1) * * As of Mar 2024, it takes <=350ms to persist a holesky state on fast server + * + * For Gloas: Processes both block state and payload state variants together. The decision of which roots to persist/prune + * is based on root canonicality (from state's view), not payload presence. Both variants are managed as a unit. */ async processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise { let persistCount = 0; @@ -549,7 +587,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * * Use seed state from the block cache if cannot find any seed states within this cache. */ - findSeedStateToReload(reloadedCp: CheckpointHex): CachedBeaconStateAllForks { + findSeedStateToReload(reloadedCp: CheckpointHexPayload): CachedBeaconStateAllForks { const maxEpoch = Math.max(...Array.from(this.epochIndex.keys())); const reloadedCpSlot = computeStartSlotAtEpoch(reloadedCp.epoch); let firstState: CachedBeaconStateAllForks | null = null; @@ -562,32 +600,35 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { return firstState; } - for (const rootHex of this.epochIndex.get(epoch) || []) { - const cpKey = toCacheKey({rootHex, epoch}); - const cacheItem = this.cache.get(cpKey); - if (cacheItem === undefined) { - // should not happen - continue; - } - if (isInMemoryCacheItem(cacheItem)) { - const {state} = cacheItem; - if (firstState === null) { - firstState = state; + for (const [rootHex, bitmask] of this.epochIndex.get(epoch) || []) { + for (const flag of PAYLOAD_AVAILABILITY_ALL) { + if (!(bitmask & flag)) continue; + const payloadPresent = fromPayloadAvailability(flag); + const cpKey = toCacheKey({rootHex, epoch, payloadPresent}); + const cacheItem = this.cache.get(cpKey); + if (cacheItem === undefined) { + continue; } - const cpLog = {cpEpoch: epoch, cpRoot: rootHex}; - - try { - // amongst states of the same epoch, choose the one with the same view of reloadedCp - if ( - reloadedCpSlot < state.slot && - toRootHex(getBlockRootAtSlot(state, reloadedCpSlot)) === reloadedCp.rootHex - ) { - this.logger.verbose("Reload: use checkpoint state as seed state", {...cpLog, ...logCtx}); - return state; + if (isInMemoryCacheItem(cacheItem)) { + const {state} = cacheItem; + if (firstState === null) { + firstState = state; + } + const cpLog = {cpEpoch: epoch, cpRoot: rootHex, payloadPresent}; + + try { + // amongst states of the same epoch, choose the one with the same view of reloadedCp + if ( + reloadedCpSlot < state.slot && + toRootHex(getBlockRootAtSlot(state, reloadedCpSlot)) === reloadedCp.rootHex + ) { + this.logger.verbose("Reload: use checkpoint state as seed state", {...cpLog, ...logCtx}); + return state; + } + } catch (e) { + // getBlockRootAtSlot may throw error + this.logger.debug("Error finding checkpoint state to reload", {...cpLog, ...logCtx}, e as Error); } - } catch (e) { - // getBlockRootAtSlot may throw error - this.logger.debug("Error finding checkpoint state to reload", {...cpLog, ...logCtx}, e as Error); } } } @@ -604,6 +645,31 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.epochIndex.clear(); } + private addToEpochIndex(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): void { + const rootMap = this.epochIndex.getOrDefault(epoch); + rootMap.set(rootHex, (rootMap.get(rootHex) ?? 0) | toPayloadAvailability(payloadPresent)); + } + + private removeFromEpochIndex(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): void { + const rootMap = this.epochIndex.get(epoch); + if (rootMap === undefined) return; + const existing = rootMap.get(rootHex); + if (existing === undefined) return; + const updated = existing & ~toPayloadAvailability(payloadPresent); + if (updated === 0) { + rootMap.delete(rootHex); + if (rootMap.size === 0) { + this.epochIndex.delete(epoch); + } + } else { + rootMap.set(rootHex, updated); + } + } + + private hasPayloadVariant(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): boolean { + return Boolean((this.epochIndex.get(epoch)?.get(rootHex) ?? 0) & toPayloadAvailability(payloadPresent)); + } + /** ONLY FOR DEBUGGING PURPOSES. For lodestar debug API */ dumpSummary(): routes.lodestar.StateCacheItem[] { return Array.from(this.cache.keys()).map((key) => { @@ -682,7 +748,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { const prevEpochRoot = toRootHex(getBlockRootAtSlot(state, epochBoundarySlot - 1)); // for each epoch, usually there are 2 rootHexes respective to the 2 checkpoint states: Previous Root Checkpoint State and Current Root Checkpoint State - const cpRootHexes = this.epochIndex.get(epoch) ?? []; + const cpRootHexMap = this.epochIndex.get(epoch) ?? new Map(); const persistedRootHexes = new Set(); // 1) if there is no CRCS, persist PRCS (block 0 of epoch is skipped). In this case prevEpochRoot === epochBoundaryHex @@ -691,76 +757,81 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { persistedRootHexes.add(epochBoundaryHex); // 3) persist any states with unknown roots to this state - for (const rootHex of cpRootHexes) { + for (const rootHex of cpRootHexMap.keys()) { if (rootHex !== epochBoundaryHex && rootHex !== prevEpochRoot) { persistedRootHexes.add(rootHex); } } - for (const rootHex of cpRootHexes) { - const cpKey = toCacheKey({epoch: epoch, rootHex}); - const cacheItem = this.cache.get(cpKey); - - if (cacheItem !== undefined && isInMemoryCacheItem(cacheItem)) { - let {persistedKey} = cacheItem; - const {state} = cacheItem; - const logMeta = { - stateSlot: state.slot, - rootHex, - epochBoundaryHex, - persistedKey: persistedKey ? toHex(persistedKey) : "", - }; - - if (persistedRootHexes.has(rootHex)) { - if (persistedKey) { - // we don't care if the checkpoint state is already persisted - this.logger.verbose("Pruned checkpoint state from memory but no need to persist", logMeta); - } else { - // persist and do not update epochIndex - this.metrics?.cpStateCache.statePersistSecFromSlot.observe( - this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0 - ); - const cpPersist = {epoch: epoch, root: fromHex(rootHex)}; - // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory. - // As monitored on holesky as of Jan 2024: - // - This does not increase heap allocation while gc time is the same - // - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s) - // - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s) - // - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization - const timer = this.metrics?.stateSerializeDuration.startTimer({ - source: AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, - }); - persistedKey = await serializeState( - state, - AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, - (stateBytes) => { - timer?.(); - return this.datastore.write(cpPersist, stateBytes); - }, - this.bufferPool - ); + for (const [rootHex, bitmask] of cpRootHexMap) { + for (const flag of PAYLOAD_AVAILABILITY_ALL) { + if (!(bitmask & flag)) continue; + const payloadPresent = fromPayloadAvailability(flag); + const cpKey = toCacheKey({epoch: epoch, rootHex, payloadPresent}); + const cacheItem = this.cache.get(cpKey); - persistCount++; - this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", { - ...logMeta, - persistedKey: toHex(persistedKey), - }); - } - // overwrite cpKey, this means the state is deleted from memory - this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey}); - } else { - if (persistedKey) { - // persisted file will be eventually deleted by the archive task - // this also means the state is deleted from memory + if (cacheItem !== undefined && isInMemoryCacheItem(cacheItem)) { + let {persistedKey} = cacheItem; + const {state} = cacheItem; + const logMeta = { + stateSlot: state.slot, + rootHex, + payloadPresent, + epochBoundaryHex, + persistedKey: persistedKey ? toHex(persistedKey) : "", + }; + + if (persistedRootHexes.has(rootHex)) { + if (persistedKey) { + // we don't care if the checkpoint state is already persisted + this.logger.verbose("Pruned checkpoint state from memory but no need to persist", logMeta); + } else { + // persist and do not update epochIndex + this.metrics?.cpStateCache.statePersistSecFromSlot.observe( + this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0 + ); + const cpPersist = {epoch: epoch, root: fromHex(rootHex)}; + // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory. + // As monitored on holesky as of Jan 2024: + // - This does not increase heap allocation while gc time is the same + // - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s) + // - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s) + // - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization + const timer = this.metrics?.stateSerializeDuration.startTimer({ + source: AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, + }); + persistedKey = await serializeState( + state, + AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, + (stateBytes) => { + timer?.(); + return this.datastore.write(cpPersist, stateBytes, payloadPresent); + }, + this.bufferPool + ); + + persistCount++; + this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", { + ...logMeta, + persistedKey: toHex(persistedKey), + }); + } + // overwrite cpKey, this means the state is deleted from memory this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey}); - // do not update epochIndex } else { - // delete the state from memory - this.cache.delete(cpKey); - this.epochIndex.get(epoch)?.delete(rootHex); + if (persistedKey) { + // persisted file will be eventually deleted by the archive task + // this also means the state is deleted from memory + this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey}); + // do not update epochIndex + } else { + // delete the state from memory + this.cache.delete(cpKey); + this.removeFromEpochIndex(epoch, rootHex, payloadPresent); + } + this.metrics?.cpStateCache.statePruneFromMemoryCount.inc(); + this.logger.verbose("Pruned checkpoint state from memory", logMeta); } - this.metrics?.cpStateCache.statePruneFromMemoryCount.inc(); - this.logger.verbose("Pruned checkpoint state from memory", logMeta); } } } @@ -773,26 +844,40 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { */ private async deleteAllEpochItems(epoch: Epoch): Promise { let persistCount = 0; - const rootHexes = this.epochIndex.get(epoch) || []; - for (const rootHex of rootHexes) { - const key = toCacheKey({rootHex, epoch}); - const cacheItem = this.cache.get(key); - - if (cacheItem) { - const persistedKey = isPersistedCacheItem(cacheItem) ? cacheItem.value : cacheItem.persistedKey; - if (persistedKey) { - await this.datastore.remove(persistedKey); - persistCount++; - this.metrics?.cpStateCache.persistedStateRemoveCount.inc(); + const rootHexMap = this.epochIndex.get(epoch) || new Map(); + for (const [rootHex, bitmask] of rootHexMap) { + for (const flag of PAYLOAD_AVAILABILITY_ALL) { + if (!(bitmask & flag)) continue; + const payloadPresent = fromPayloadAvailability(flag); + const key = toCacheKey({rootHex, epoch, payloadPresent}); + const cacheItem = this.cache.get(key); + + if (cacheItem) { + const persistedKey = isPersistedCacheItem(cacheItem) ? cacheItem.value : cacheItem.persistedKey; + if (persistedKey) { + await this.datastore.remove(persistedKey); + persistCount++; + this.metrics?.cpStateCache.persistedStateRemoveCount.inc(); + } } + this.cache.delete(key); + this.logger.verbose("Pruned checkpoint state", { + epoch, + rootHex, + payloadPresent, + type: cacheItem ? (isPersistedCacheItem(cacheItem) ? "persisted" : "in-memory") : "missing", + }); } - this.cache.delete(key); } this.epochIndex.delete(epoch); - this.logger.verbose("Pruned checkpoint states for epoch", { + this.logger.verbose("Pruned all checkpoint states for epoch", { epoch, persistCount, - rootHexes: Array.from(rootHexes).join(","), + items: Array.from(rootHexMap.entries()) + .flatMap(([rootHex, bitmask]) => + PAYLOAD_AVAILABILITY_ALL.filter((f) => bitmask & f).map((f) => `${rootHex}:${fromPayloadAvailability(f)}`) + ) + .join(","), }); } @@ -844,29 +929,57 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } } -export function toCheckpointHex(checkpoint: phase0.Checkpoint): CheckpointHex { +export function toCheckpointHexPayload(checkpoint: phase0.Checkpoint, payloadPresent: boolean): CheckpointHexPayload { return { epoch: checkpoint.epoch, rootHex: toRootHex(checkpoint.root), + payloadPresent, }; } -export function toCheckpointKey(cp: CheckpointHex): string { - return `${cp.rootHex}:${cp.epoch}`; -} +/** + * Convert fork-choice CheckpointWithPayloadStatus to beacon-node CheckpointHexPayload. + * Maps PayloadStatus enum to boolean payloadPresent. + * @throws Error if checkpoint has PENDING payload status (ambiguous which variant to use) + */ +export function fcCheckpointToHexPayload(checkpoint: CheckpointWithPayloadStatus): CheckpointHexPayload { + const PayloadStatus = {PENDING: 0, EMPTY: 1, FULL: 2} as const; -function toCacheKey(cp: CheckpointHex | phase0.Checkpoint): CacheKey { - if (isCheckpointHex(cp)) { - return `${cp.rootHex}_${cp.epoch}`; + if (checkpoint.payloadStatus === PayloadStatus.PENDING) { + throw Error( + `Cannot convert checkpoint with PENDING payload status at epoch ${checkpoint.epoch} root ${checkpoint.rootHex}` + ); } - return `${toRootHex(cp.root)}_${cp.epoch}`; + + return { + epoch: checkpoint.epoch, + rootHex: checkpoint.rootHex, + payloadPresent: checkpoint.payloadStatus === PayloadStatus.FULL, + }; +} + +export function toCheckpointKey(cp: CheckpointHexPayload): string { + return `${cp.rootHex}:${cp.epoch}:${cp.payloadPresent}`; } -function fromCacheKey(key: CacheKey): CheckpointHex { - const [rootHex, epoch] = key.split("_"); +/** + * Convert checkpoint to cache key string. + * Format: `{rootHex}_{epoch}_{payloadPresent}` + */ +function toCacheKey(cp: CheckpointHexPayload): CacheKey { + return `${cp.rootHex}_${cp.epoch}_${cp.payloadPresent}`; +} + +function fromCacheKey(key: CacheKey): CheckpointHexPayload { + const parts = key.split("_"); + const rootHex = parts[0]; + const epoch = Number(parts[1]); + // For backward compatibility with old format (rootHex_epoch), default to true + const payloadPresent = parts.length > 2 ? parts[2] === "true" : true; return { rootHex, - epoch: Number(epoch), + epoch, + payloadPresent, }; } @@ -883,7 +996,3 @@ function isInMemoryCacheItem(cacheItem: CacheItem): cacheItem is InMemoryCacheIt function isPersistedCacheItem(cacheItem: CacheItem): cacheItem is PersistedCacheItem { return cacheItem.type === CacheItemType.persisted; } - -function isCheckpointHex(cp: CheckpointHex | phase0.Checkpoint): cp is CheckpointHex { - return (cp as CheckpointHex).rootHex !== undefined; -} diff --git a/packages/beacon-node/src/chain/stateCache/types.ts b/packages/beacon-node/src/chain/stateCache/types.ts index b16590967c9d..7fc15e31e6df 100644 --- a/packages/beacon-node/src/chain/stateCache/types.ts +++ b/packages/beacon-node/src/chain/stateCache/types.ts @@ -2,7 +2,11 @@ import {routes} from "@lodestar/api"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {Epoch, RootHex, phase0} from "@lodestar/types"; -export type CheckpointHex = {epoch: Epoch; rootHex: RootHex}; +/** + * Checkpoint hex representation for state cache keys. + * Extends CheckpointWithHex (from fork-choice) with payloadPresent. + */ +export type CheckpointHexPayload = {epoch: Epoch; rootHex: RootHex; payloadPresent: boolean}; /** * Lodestar currently keeps two state caches around. @@ -31,6 +35,8 @@ export interface BlockStateCache { size: number; prune(headStateRootHex: RootHex): void; deleteAllBeforeEpoch(finalizedEpoch: Epoch): void; + /** Upgrade cache capacity for Gloas fork (2x states for block + payload states) */ + upgradeToGloas(): void; dumpSummary(): routes.lodestar.StateCacheItem[]; /** Expose beacon states stored in cache. Use with caution */ getStates(): IterableIterator; @@ -59,13 +65,17 @@ export interface BlockStateCache { */ export interface CheckpointStateCache { init?: () => Promise; - getOrReload(cp: CheckpointHex): Promise; - getStateOrBytes(cp: CheckpointHex): Promise; - get(cpOrKey: CheckpointHex | string): CachedBeaconStateAllForks | null; - add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks): void; - getLatest(rootHex: RootHex, maxEpoch: Epoch): CachedBeaconStateAllForks | null; - getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch): Promise; - updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null; + getOrReload(cp: CheckpointHexPayload): Promise; + getStateOrBytes(cp: CheckpointHexPayload): Promise; + get(cpOrKey: CheckpointHexPayload | string): CachedBeaconStateAllForks | null; + add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks, payloadPresent: boolean): void; + getLatest(rootHex: RootHex, maxEpoch: Epoch, payloadPresent: boolean): CachedBeaconStateAllForks | null; + getOrReloadLatest( + rootHex: RootHex, + maxEpoch: Epoch, + payloadPresent: boolean + ): Promise; + updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null; prune(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void; pruneFinalized(finalizedEpoch: Epoch): void; processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise; diff --git a/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts b/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts index 9dd74e170f29..4b2ada3b872a 100644 --- a/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts +++ b/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts @@ -10,9 +10,9 @@ import {FIFOBlockStateCache} from "../../../../src/chain/index.js"; import {checkpointToDatastoreKey} from "../../../../src/chain/stateCache/datastore/index.js"; import { PersistentCheckpointStateCache, - toCheckpointHex, + toCheckpointHexPayload, } from "../../../../src/chain/stateCache/persistentCheckpointsCache.js"; -import {CheckpointHex} from "../../../../src/chain/stateCache/types.js"; +import {CheckpointHexPayload} from "../../../../src/chain/stateCache/types.js"; import {getTestDatastore} from "../../../utils/chain/stateCache/datastore.js"; import {generateCachedState} from "../../../utils/state.js"; @@ -23,7 +23,10 @@ describe("PersistentCheckpointStateCache", () => { let root0a: Buffer, root0b: Buffer, root1: Buffer, root2: Buffer; let cp0a: phase0.Checkpoint, cp0b: phase0.Checkpoint, cp1: phase0.Checkpoint, cp2: phase0.Checkpoint; - let cp0aHex: CheckpointHex, cp0bHex: CheckpointHex, cp1Hex: CheckpointHex, cp2Hex: CheckpointHex; + let cp0aHex: CheckpointHexPayload, + cp0bHex: CheckpointHexPayload, + cp1Hex: CheckpointHexPayload, + cp2Hex: CheckpointHexPayload; let persistent0bKey: RootHex; const startSlotEpoch20 = computeStartSlotAtEpoch(20); const startSlotEpoch21 = computeStartSlotAtEpoch(21); @@ -53,8 +56,8 @@ describe("PersistentCheckpointStateCache", () => { cp0b = {epoch: 20, root: root0b}; cp1 = {epoch: 21, root: root1}; cp2 = {epoch: 22, root: root2}; - [cp0aHex, cp0bHex, cp1Hex, cp2Hex] = [cp0a, cp0b, cp1, cp2].map((cp) => toCheckpointHex(cp)); - persistent0bKey = toHexString(checkpointToDatastoreKey(cp0b)); + [cp0aHex, cp0bHex, cp1Hex, cp2Hex] = [cp0a, cp0b, cp1, cp2].map((cp) => toCheckpointHexPayload(cp, true)); + persistent0bKey = toHexString(checkpointToDatastoreKey(cp0b, true)); const allStates = [cp0a, cp0b, cp1, cp2] .map((cp) => generateCachedState({slot: cp.epoch * SLOTS_PER_EPOCH})) .map((state, i) => { @@ -104,28 +107,30 @@ describe("PersistentCheckpointStateCache", () => { }, {maxCPStateEpochsInMemory: 2} ); - cache.add(cp0a, states["cp0a"]); - cache.add(cp0b, states["cp0b"]); - cache.add(cp1, states["cp1"]); + cache.add(cp0a, states["cp0a"], true); + cache.add(cp0b, states["cp0b"], true); + cache.add(cp1, states["cp1"], true); }); it("getLatest", () => { // cp0 - expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch)?.hashTreeRoot()).toEqual(states["cp0a"].hashTreeRoot()); - expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch + 1)?.hashTreeRoot()).toEqual(states["cp0a"].hashTreeRoot()); - expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch - 1)?.hashTreeRoot()).toBeUndefined(); + expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch, true)?.hashTreeRoot()).toEqual(states["cp0a"].hashTreeRoot()); + expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch + 1, true)?.hashTreeRoot()).toEqual( + states["cp0a"].hashTreeRoot() + ); + expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch - 1, true)?.hashTreeRoot()).toBeUndefined(); // cp1 - expect(cache.getLatest(cp1Hex.rootHex, cp1.epoch)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); - expect(cache.getLatest(cp1Hex.rootHex, cp1.epoch + 1)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); - expect(cache.getLatest(cp1Hex.rootHex, cp1.epoch - 1)?.hashTreeRoot()).toBeUndefined(); + expect(cache.getLatest(cp1Hex.rootHex, cp1.epoch, true)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); + expect(cache.getLatest(cp1Hex.rootHex, cp1.epoch + 1, true)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); + expect(cache.getLatest(cp1Hex.rootHex, cp1.epoch - 1, true)?.hashTreeRoot()).toBeUndefined(); // cp2 - expect(cache.getLatest(cp2Hex.rootHex, cp2.epoch)?.hashTreeRoot()).toBeUndefined(); + expect(cache.getLatest(cp2Hex.rootHex, cp2.epoch, true)?.hashTreeRoot()).toBeUndefined(); }); it("getOrReloadLatest", async () => { - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); // cp0b is persisted @@ -133,19 +138,21 @@ describe("PersistentCheckpointStateCache", () => { expect(Array.from(fileApisBuffer.keys())).toEqual([persistent0bKey]); // getLatest() does not reload from disk - expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch)).toBeNull(); - expect(cache.getLatest(cp0bHex.rootHex, cp0b.epoch)).toBeNull(); + expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch, true)).toBeNull(); + expect(cache.getLatest(cp0bHex.rootHex, cp0b.epoch, true)).toBeNull(); // cp0a has the root from previous epoch so we only prune it from db - expect(await cache.getOrReloadLatest(cp0aHex.rootHex, cp0a.epoch)).toBeNull(); + expect(await cache.getOrReloadLatest(cp0aHex.rootHex, cp0a.epoch, true)).toBeNull(); // but getOrReloadLatest() does for cp0b - expect((await cache.getOrReloadLatest(cp0bHex.rootHex, cp0b.epoch))?.serialize()).toEqual(stateBytes["cp0b"]); - expect((await cache.getOrReloadLatest(cp0bHex.rootHex, cp0b.epoch + 1))?.serialize()).toEqual(stateBytes["cp0b"]); - expect((await cache.getOrReloadLatest(cp0bHex.rootHex, cp0b.epoch - 1))?.serialize()).toBeUndefined(); + expect((await cache.getOrReloadLatest(cp0bHex.rootHex, cp0b.epoch, true))?.serialize()).toEqual(stateBytes["cp0b"]); + expect((await cache.getOrReloadLatest(cp0bHex.rootHex, cp0b.epoch + 1, true))?.serialize()).toEqual( + stateBytes["cp0b"] + ); + expect((await cache.getOrReloadLatest(cp0bHex.rootHex, cp0b.epoch - 1, true))?.serialize()).toBeUndefined(); }); it("pruneFinalized and getStateOrBytes", async () => { - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(((await cache.getStateOrBytes(cp0bHex)) as CachedBeaconStateAllForks).hashTreeRoot()).toEqual( states["cp0b"].hashTreeRoot() ); @@ -179,9 +186,9 @@ describe("PersistentCheckpointStateCache", () => { }, {maxCPStateEpochsInMemory: 2} ); - cache.add(cp0a, states["cp0a"]); - cache.add(cp0b, states["cp0b"]); - cache.add(cp1, states["cp1"]); + cache.add(cp0a, states["cp0a"], true); + cache.add(cp0b, states["cp0b"], true); + cache.add(cp1, states["cp1"], true); }); // epoch: 19 20 21 22 23 @@ -192,7 +199,7 @@ describe("PersistentCheckpointStateCache", () => { // | // 0a it("single state at lowest memory epoch", async () => { - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); expect(cache.findSeedStateToReload(cp0aHex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(cache.findSeedStateToReload(cp0bHex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); @@ -208,7 +215,7 @@ describe("PersistentCheckpointStateCache", () => { // ^ ^ // cp1a={0a, 21} {0a, 22}=cp2a it("multiple states at lowest memory epoch", async () => { - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); const cp1a = {epoch: 21, root: root0a}; @@ -216,14 +223,14 @@ describe("PersistentCheckpointStateCache", () => { cp1aState.slot = 21 * SLOTS_PER_EPOCH; cp1aState.blockRoots.set(startSlotEpoch21 % SLOTS_PER_HISTORICAL_ROOT, root0a); cp1aState.commit(); - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); const cp2a = {epoch: 22, root: root0a}; const cp2aState = cp1aState.clone(); cp2aState.slot = 22 * SLOTS_PER_EPOCH; cp2aState.blockRoots.set(startSlotEpoch22 % SLOTS_PER_HISTORICAL_ROOT, root0a); cp2aState.commit(); - cache.add(cp2a, cp2aState); + cache.add(cp2a, cp2aState, true); const root3 = Buffer.alloc(32, 100); const state3 = cp2aState.clone(); @@ -237,9 +244,9 @@ describe("PersistentCheckpointStateCache", () => { expect(cache.findSeedStateToReload(cp0bHex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); const randomRoot = Buffer.alloc(32, 101); // for other random root it'll pick the first state of epoch 21 which is states["cp1"] - expect(cache.findSeedStateToReload({epoch: 20, rootHex: toHexString(randomRoot)})?.hashTreeRoot()).toEqual( - states["cp1"].hashTreeRoot() - ); + expect( + cache.findSeedStateToReload({epoch: 20, rootHex: toHexString(randomRoot), payloadPresent: true})?.hashTreeRoot() + ).toEqual(states["cp1"].hashTreeRoot()); }); }); @@ -256,9 +263,9 @@ describe("PersistentCheckpointStateCache", () => { }, {maxCPStateEpochsInMemory: 2} ); - cache.add(cp0a, states["cp0a"]); - cache.add(cp0b, states["cp0b"]); - cache.add(cp1, states["cp1"]); + cache.add(cp0a, states["cp0a"], true); + cache.add(cp0b, states["cp0b"], true); + cache.add(cp1, states["cp1"], true); }); // epoch: 19 20 21 22 23 @@ -270,7 +277,7 @@ describe("PersistentCheckpointStateCache", () => { // 0a it("no reorg", async () => { expect(fileApisBuffer.size).toEqual(0); - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); expect(cache.get(cp2Hex)?.hashTreeRoot()).toEqual(states["cp2"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -305,7 +312,7 @@ describe("PersistentCheckpointStateCache", () => { it("reorg in same epoch", async () => { // mostly the same to the above test expect(fileApisBuffer.size).toEqual(0); - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); expect(cache.get(cp2Hex)?.hashTreeRoot()).toEqual(states["cp2"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -349,7 +356,7 @@ describe("PersistentCheckpointStateCache", () => { // {1a, 22}=cp2a it("reorg 1 epoch", async () => { // process root2 state - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); @@ -361,7 +368,7 @@ describe("PersistentCheckpointStateCache", () => { // assuming reorg block is at slot 5 of epoch 21 cp2aState.blockRoots.set((startSlotEpoch21 + 5) % SLOTS_PER_HISTORICAL_ROOT, root1a); cp2aState.blockRoots.set(startSlotEpoch22 % SLOTS_PER_HISTORICAL_ROOT, root1a); - cache.add(cp2a, cp2aState); + cache.add(cp2a, cp2aState, true); // block state of root3 in epoch 22 is built on cp2a const blockStateRoot3 = cp2aState.clone(); @@ -373,7 +380,7 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); // epoch 22 has 2 checkpoint states expect(cache.get(cp2Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp2a))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp2a, true))).not.toBeNull(); // epoch 21 has 1 checkpoint state expect(cache.get(cp1Hex)).not.toBeNull(); // epoch 20 has 0 checkpoint state @@ -393,12 +400,14 @@ describe("PersistentCheckpointStateCache", () => { // cp1a={0a, 21} {0a, 22}=cp2a it("reorg 2 epochs", async () => { // process root2 state - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); // reload cp0b from disk - expect((await cache.getOrReload(toCheckpointHex(cp0b)))?.serialize()).toStrictEqual(stateBytes["cp0b"]); + expect((await cache.getOrReload(toCheckpointHexPayload(cp0b, true)))?.serialize()).toStrictEqual( + stateBytes["cp0b"] + ); // regen generates cp1a const root0a = Buffer.alloc(32, 100); @@ -407,14 +416,14 @@ describe("PersistentCheckpointStateCache", () => { cp1aState.slot = 21 * SLOTS_PER_EPOCH; // assuming reorg block is at slot 5 of epoch 20 cp1aState.blockRoots.set((startSlotEpoch20 + 5) % SLOTS_PER_HISTORICAL_ROOT, root0a); - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); // regen generates cp2a const cp2a = {epoch: 22, root: root0a}; const cp2aState = cp1aState.clone(); cp2aState.slot = 22 * SLOTS_PER_EPOCH; cp2aState.blockRoots.set(startSlotEpoch22 % SLOTS_PER_HISTORICAL_ROOT, root0a); - cache.add(cp2a, cp2aState); + cache.add(cp2a, cp2aState, true); // block state of root3 in epoch 22 is built on cp2a const blockStateRoot3 = cp2aState.clone(); @@ -426,9 +435,9 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); // epoch 21 and 22 have 2 checkpoint states expect(cache.get(cp1Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1a))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1a, true))).not.toBeNull(); expect(cache.get(cp2Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp2a))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp2a, true))).not.toBeNull(); // epoch 20 has 0 checkpoint state expect(cache.get(cp0aHex)).toBeNull(); expect(cache.get(cp0bHex)).toBeNull(); @@ -446,28 +455,28 @@ describe("PersistentCheckpointStateCache", () => { // cp1a={0a, 21} {0a, 22}=cp2a it("reorg 3 epochs, persist cp 0a", async () => { // process root2 state - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); // cp0a was pruned from memory and not in disc expect(await cache.getStateOrBytes(cp0aHex)).toBeNull(); // regen needs to regen cp0a - cache.add(cp0a, states["cp0a"]); + cache.add(cp0a, states["cp0a"], true); // regen generates cp1a const cp1a = {epoch: 21, root: root0a}; const cp1aState = generateCachedState({slot: 21 * SLOTS_PER_EPOCH}); cp1aState.blockRoots.set((startSlotEpoch20 - 1) % SLOTS_PER_HISTORICAL_ROOT, root0a); cp1aState.blockRoots.set(startSlotEpoch20 % SLOTS_PER_HISTORICAL_ROOT, root0a); - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); // regen generates cp2a const cp2a = {epoch: 22, root: root0a}; const cp2aState = cp1aState.clone(); cp2aState.slot = 22 * SLOTS_PER_EPOCH; cp2aState.blockRoots.set(startSlotEpoch21 % SLOTS_PER_HISTORICAL_ROOT, root0a); - cache.add(cp2a, cp2aState); + cache.add(cp2a, cp2aState, true); // block state of root3 in epoch 22 is built on cp2a const blockStateRoot3 = cp2aState.clone(); @@ -482,9 +491,9 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([cp0b, cp0a], [stateBytes["cp0b"], stateBytes["cp0a"]]); // epoch 21 and 22 have 2 checkpoint states expect(cache.get(cp1Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1a))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1a, true))).not.toBeNull(); expect(cache.get(cp2Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp2a))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp2a, true))).not.toBeNull(); // epoch 20 has 0 checkpoint state expect(cache.get(cp0aHex)).toBeNull(); expect(cache.get(cp0bHex)).toBeNull(); @@ -502,14 +511,14 @@ describe("PersistentCheckpointStateCache", () => { // cp1b={0b, 21} {0b, 22}=cp2b it("reorg 3 epochs, prune but no persist", async () => { // process root2 state - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); // cp0a was pruned from memory and not in disc expect(await cache.getStateOrBytes(cp0aHex)).toBeNull(); // regen needs to reload cp0b - cache.add(cp0b, states["cp0b"]); + cache.add(cp0b, states["cp0b"], true); expect(((await cache.getStateOrBytes(cp0bHex)) as CachedBeaconStateAllForks).hashTreeRoot()).toEqual( states["cp0b"].hashTreeRoot() ); @@ -519,14 +528,14 @@ describe("PersistentCheckpointStateCache", () => { const cp1bState = states["cp0b"].clone(); cp1bState.slot = 21 * SLOTS_PER_EPOCH; cp1bState.blockRoots.set(startSlotEpoch21 % SLOTS_PER_HISTORICAL_ROOT, root0b); - cache.add(cp1b, cp1bState); + cache.add(cp1b, cp1bState, true); // regen generates cp2b const cp2b = {epoch: 22, root: root0b}; const cp2bState = cp1bState.clone(); cp2bState.slot = 22 * SLOTS_PER_EPOCH; cp2bState.blockRoots.set(startSlotEpoch22 % SLOTS_PER_HISTORICAL_ROOT, root0b); - cache.add(cp2b, cp2bState); + cache.add(cp2b, cp2bState, true); // block state of root3 in epoch 22 is built on cp2a const blockStateRoot3 = cp2bState.clone(); @@ -540,9 +549,9 @@ describe("PersistentCheckpointStateCache", () => { // epoch 21 and 22 have 2 checkpoint states expect(cache.get(cp1Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1b))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1b, true))).not.toBeNull(); expect(cache.get(cp2Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp2b))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp2b, true))).not.toBeNull(); // epoch 20 has 0 checkpoint state expect(cache.get(cp0aHex)).toBeNull(); expect(cache.get(cp0bHex)).toBeNull(); @@ -562,8 +571,8 @@ describe("PersistentCheckpointStateCache", () => { }, {maxCPStateEpochsInMemory: 1} ); - cache.add(cp0a, states["cp0a"]); - cache.add(cp0b, states["cp0b"]); + cache.add(cp0a, states["cp0a"], true); + cache.add(cp0b, states["cp0b"], true); }); // epoch: 19 20 21 22 23 @@ -575,7 +584,7 @@ describe("PersistentCheckpointStateCache", () => { // 0a it("no reorg", async () => { expect(fileApisBuffer.size).toEqual(0); - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -610,7 +619,7 @@ describe("PersistentCheckpointStateCache", () => { it("reorg in same epoch", async () => { // almost the same to "no reorg" test expect(fileApisBuffer.size).toEqual(0); - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -660,7 +669,7 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([], []); // cp1 - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -671,7 +680,7 @@ describe("PersistentCheckpointStateCache", () => { const cp1aState = state1a.clone(); cp1aState.slot = 21 * SLOTS_PER_EPOCH; const cp1a = {epoch: 21, root: root1a}; - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); const blockStateRoot2 = cp1aState.clone(); blockStateRoot2.slot = 21 * SLOTS_PER_EPOCH + 3; const root2 = Buffer.alloc(32, 100); @@ -680,8 +689,8 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); // keep these 2 cp states at epoch 21 - expect(cache.get(toCheckpointHex(cp1a))).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1a, true))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1, true))).not.toBeNull(); }); // epoch: 19 20 21 22 23 @@ -694,7 +703,7 @@ describe("PersistentCheckpointStateCache", () => { it("reorg 1 epoch, no persist 0b", async () => { expect(fileApisBuffer.size).toEqual(0); // cp1 - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -702,7 +711,7 @@ describe("PersistentCheckpointStateCache", () => { expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); // simulate regen - cache.add(cp0b, states["cp0b"]); + cache.add(cp0b, states["cp0b"], true); expect(((await cache.getStateOrBytes(cp0bHex)) as CachedBeaconStateAllForks).hashTreeRoot()).toEqual( states["cp0b"].hashTreeRoot() ); @@ -710,7 +719,7 @@ describe("PersistentCheckpointStateCache", () => { const cp1bState = states["cp0b"].clone(); cp1bState.slot = 21 * SLOTS_PER_EPOCH; const cp1b = {epoch: 21, root: root0b}; - cache.add(cp1b, cp1bState); + cache.add(cp1b, cp1bState, true); const blockStateRoot2 = cp1bState.clone(); blockStateRoot2.slot = 21 * SLOTS_PER_EPOCH + 3; const root2 = Buffer.alloc(32, 100); @@ -720,8 +729,8 @@ describe("PersistentCheckpointStateCache", () => { // but cp0b in-memory state is pruned expect(await cache.getStateOrBytes(cp0bHex)).toEqual(stateBytes["cp0b"]); // keep these 2 cp states at epoch 21 - expect(cache.get(toCheckpointHex(cp1b))).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1b, true))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1, true))).not.toBeNull(); }); // epoch: 19 20 21 22 23 @@ -748,7 +757,7 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([], []); // cp1 - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -762,11 +771,11 @@ describe("PersistentCheckpointStateCache", () => { expect(await cache.getStateOrBytes(cp0aHex)).toBeNull(); // root2, regen cp0a - cache.add(cp0a, states["cp0a"]); + cache.add(cp0a, states["cp0a"], true); const cp1aState = state1a.clone(); cp1aState.slot = 21 * SLOTS_PER_EPOCH; const cp1a = {epoch: 21, root: root1a}; - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); const blockStateRoot2 = cp1aState.clone(); blockStateRoot2.slot = 21 * SLOTS_PER_EPOCH + 3; const root2 = Buffer.alloc(32, 100); @@ -775,8 +784,8 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([cp0b, cp0a], [stateBytes["cp0b"], stateBytes["cp0a"]]); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); // keep these 2 cp states at epoch 21 - expect(cache.get(toCheckpointHex(cp1a))).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1a, true))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1, true))).not.toBeNull(); }); // epoch: 19 20 21 22 23 @@ -790,7 +799,7 @@ describe("PersistentCheckpointStateCache", () => { // cp1a={0a, 21} it("reorg 2 epochs", async () => { // cp1 - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -804,11 +813,11 @@ describe("PersistentCheckpointStateCache", () => { expect(await cache.getStateOrBytes(cp0aHex)).toBeNull(); // root2, regen cp0a - cache.add(cp0a, states["cp0a"]); + cache.add(cp0a, states["cp0a"], true); const cp1aState = states["cp0a"].clone(); cp1aState.slot = 21 * SLOTS_PER_EPOCH; const cp1a = {epoch: 21, root: root0a}; - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); const blockStateRoot2 = cp1aState.clone(); blockStateRoot2.slot = 21 * SLOTS_PER_EPOCH + 3; const root2 = Buffer.alloc(32, 100); @@ -817,8 +826,8 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([cp0b, cp0a], [stateBytes["cp0b"], stateBytes["cp0a"]]); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); // keep these 2 cp states at epoch 21 - expect(cache.get(toCheckpointHex(cp1a))).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1a, true))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1, true))).not.toBeNull(); }); describe("processState, maxEpochsInMemory = 0", () => { @@ -834,8 +843,8 @@ describe("PersistentCheckpointStateCache", () => { }, {maxCPStateEpochsInMemory: 0} ); - cache.add(cp0a, states["cp0a"]); - cache.add(cp0b, states["cp0b"]); + cache.add(cp0a, states["cp0a"], true); + cache.add(cp0b, states["cp0b"], true); }); // epoch: 19 20 21 22 23 @@ -886,7 +895,7 @@ describe("PersistentCheckpointStateCache", () => { expect(await cache.getStateOrBytes(cp0bHex)).toEqual(stateBytes["cp0b"]); // simulate reload cp1b - cache.add(cp0b, states["cp0b"]); + cache.add(cp0b, states["cp0b"], true); expect(((await cache.getStateOrBytes(cp0bHex)) as CachedBeaconStateAllForks).hashTreeRoot()).toEqual( states["cp0b"].hashTreeRoot() ); @@ -931,7 +940,7 @@ describe("PersistentCheckpointStateCache", () => { state1a.slot = 20 * SLOTS_PER_EPOCH + SLOTS_PER_EPOCH + 3; state1a.blockRoots.set(state1a.slot % SLOTS_PER_HISTORICAL_ROOT, root1a); // state transition add to cache - cache.add(cp0b, states["cp0b"]); + cache.add(cp0b, states["cp0b"], true); // do not processState root1a because it's late // no need to reload cp0b because it's available in block state @@ -940,7 +949,7 @@ describe("PersistentCheckpointStateCache", () => { state1b.slot = state1a.slot + 1; state1b.blockRoots.set(state1b.slot % SLOTS_PER_HISTORICAL_ROOT, root1b); // state transition add to cache - cache.add(cp0a, states["cp0a"]); + cache.add(cp0a, states["cp0a"], true); // need to persist 2 checkpoint states expect(await cache.processState(toHexString(root1b), state1b)).toEqual(2); @@ -978,7 +987,7 @@ describe("PersistentCheckpointStateCache", () => { state1b.slot = state1a.slot + 1; state1b.blockRoots.set(state1b.slot % SLOTS_PER_HISTORICAL_ROOT, root1b); // regen should reload cp0a from disk - cache.add(cp0a, states["cp0a"]); + cache.add(cp0a, states["cp0a"], true); expect(await cache.processState(toHexString(root1b), state1b)).toEqual(1); await assertPersistedCheckpointState([cp0b, cp0a], [stateBytes["cp0b"], stateBytes["cp0a"]]); @@ -1002,18 +1011,18 @@ describe("PersistentCheckpointStateCache", () => { expect(await cache.getStateOrBytes(cp0aHex)).toBeNull(); expect(await cache.getStateOrBytes(cp0bHex)).toEqual(stateBytes["cp0b"]); - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); await assertPersistedCheckpointState([cp0b, cp1], [stateBytes["cp0b"], stateBytes["cp1"]]); // regen should populate cp0a and cp1a checkpoint states - cache.add(cp0a, states["cp0a"]); + cache.add(cp0a, states["cp0a"], true); const cp1a = {epoch: 21, root: root0a}; const cp1aState = states["cp0a"].clone(); cp1aState.blockRoots.set((20 * SLOTS_PER_EPOCH) % SLOTS_PER_HISTORICAL_ROOT, root0a); cp1aState.blockRoots.set((21 * SLOTS_PER_EPOCH) % SLOTS_PER_HISTORICAL_ROOT, root0a); cp1aState.slot = 21 * SLOTS_PER_EPOCH; - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); const root2 = Buffer.alloc(32, 100); const state2 = cp1aState.clone(); @@ -1030,13 +1039,13 @@ describe("PersistentCheckpointStateCache", () => { }); async function assertPersistedCheckpointState(cps: phase0.Checkpoint[], stateBytesArr: Uint8Array[]): Promise { - const persistedKeys = cps.map((cp) => toHexString(checkpointToDatastoreKey(cp))); + const persistedKeys = cps.map((cp) => toHexString(checkpointToDatastoreKey(cp, true))); expect(Array.from(fileApisBuffer.keys())).toStrictEqual(persistedKeys); for (const [i, persistedKey] of persistedKeys.entries()) { expect(fileApisBuffer.get(persistedKey)).toStrictEqual(stateBytesArr[i]); } for (const [i, cp] of cps.entries()) { - const cpHex = toCheckpointHex(cp); + const cpHex = toCheckpointHexPayload(cp, true); expect(await cache.getStateOrBytes(cpHex)).toStrictEqual(stateBytesArr[i]); // simple get() does not reload from disk expect(cache.get(cpHex)).toBeNull(); diff --git a/packages/beacon-node/test/unit/chain/regen/regen.test.ts b/packages/beacon-node/test/unit/chain/regen/regen.test.ts index b2e860d1a17c..1e8ace689d7f 100644 --- a/packages/beacon-node/test/unit/chain/regen/regen.test.ts +++ b/packages/beacon-node/test/unit/chain/regen/regen.test.ts @@ -85,9 +85,9 @@ describe("regen", () => { {maxCPStateEpochsInMemory: 2} ); - cache.add(cp0a, states["cp0a"]); - cache.add(cp0b, states["cp0b"]); - cache.add(cp1, states["cp1"]); + cache.add(cp0a, states["cp0a"], true); + cache.add(cp0b, states["cp0b"], true); + cache.add(cp1, states["cp1"], true); }); /** diff --git a/packages/beacon-node/test/utils/chain/stateCache/datastore.ts b/packages/beacon-node/test/utils/chain/stateCache/datastore.ts index 20d1708c1045..c4bb52acc43c 100644 --- a/packages/beacon-node/test/utils/chain/stateCache/datastore.ts +++ b/packages/beacon-node/test/utils/chain/stateCache/datastore.ts @@ -3,8 +3,8 @@ import {CPStateDatastore, checkpointToDatastoreKey} from "../../../../src/chain/ export function getTestDatastore(fileApisBuffer: Map): CPStateDatastore { const datastore: CPStateDatastore = { - write: (cp, stateBytes) => { - const persistentKey = checkpointToDatastoreKey(cp); + write: (cp, stateBytes, payloadPresent) => { + const persistentKey = checkpointToDatastoreKey(cp, payloadPresent); const stringKey = toHexString(persistentKey); if (!fileApisBuffer.has(stringKey)) { fileApisBuffer.set(stringKey, stateBytes); diff --git a/packages/beacon-node/test/utils/node/simTest.ts b/packages/beacon-node/test/utils/node/simTest.ts index e8a862bc3da6..8af87d6ebe07 100644 --- a/packages/beacon-node/test/utils/node/simTest.ts +++ b/packages/beacon-node/test/utils/node/simTest.ts @@ -63,9 +63,11 @@ export function simTestInfoTracker(bn: BeaconNode, logger: Logger): () => void { if (checkpoint.epoch <= lastSeenEpoch) return; lastSeenEpoch = checkpoint.epoch; + // Pre-Gloas: payloadPresent is always true (execution payload embedded in block) const checkpointState = bn.chain.regen.getCheckpointStateSync({ ...checkpoint, rootHex: toRootHex(checkpoint.root), + payloadPresent: true, }); if (checkpointState == null) { throw Error(`Checkpoint state not found for epoch ${checkpoint.epoch} root ${toRootHex(checkpoint.root)}`); diff --git a/packages/fork-choice/src/index.ts b/packages/fork-choice/src/index.ts index fd141bf7d70e..cf4f75bceffe 100644 --- a/packages/fork-choice/src/index.ts +++ b/packages/fork-choice/src/index.ts @@ -38,5 +38,5 @@ export type { ProtoBlock, ProtoNode, } from "./protoArray/interface.js"; -export {ExecutionStatus, PayloadStatus} from "./protoArray/interface.js"; +export {ExecutionStatus, PayloadStatus, isGloasBlock} from "./protoArray/interface.js"; export {ProtoArray} from "./protoArray/protoArray.js";