Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
8744f60
feat: async block import persistence
wemeetagain Jan 23, 2026
58cc8fb
chore: remove eagerPersistBlock
wemeetagain Jan 23, 2026
16f0733
chore: improve comments
wemeetagain Jan 23, 2026
a1df1d9
chore: simplify
wemeetagain Jan 23, 2026
958cbdc
chore: add support for serving pending blob sidecars
wemeetagain Jan 23, 2026
2ecf716
chore: add utility get sidecar chain methods
wemeetagain Jan 23, 2026
12c1cf4
chore: use chain methods to serve sidecars
wemeetagain Jan 23, 2026
05b0403
Merge branch 'unstable' into cayman/unfinalized-block-write-queue
wemeetagain Jan 23, 2026
d5931b0
chore: add comment
wemeetagain Jan 23, 2026
0176e92
chore: getBlobSidecars only return all blobs
wemeetagain Jan 23, 2026
9f89be2
chore: lint fix
wemeetagain Jan 23, 2026
42ab4a2
chore: don't await persist job in importBlock
wemeetagain Jan 27, 2026
e1b58ba
chore: add unfinalized block writes queue metrics
wemeetagain Jan 27, 2026
612a281
chore: add error log for persistBlockInputs
wemeetagain Jan 27, 2026
691afda
chore: add block slot and root to error messages
wemeetagain Jan 27, 2026
49959c9
chore: use serialized cache to get serialized column sidecars
wemeetagain Jan 27, 2026
9a843af
chore: more work on reqresp handlers
wemeetagain Jan 27, 2026
45cf2f2
chore: add one more comment
wemeetagain Jan 27, 2026
0faea9a
chore: remove block input prune-on-success
wemeetagain Jan 27, 2026
59d7a30
Merge branch 'unstable' into cayman/unfinalized-block-write-queue
wemeetagain Jan 27, 2026
e16cb91
chore: fix download_dashboards script (install its dependency)
wemeetagain Jan 27, 2026
83952f0
chore: add dashboard panels for new metrics
wemeetagain Jan 27, 2026
f2900c5
chore: graceful shutdown
wemeetagain Jan 27, 2026
5125497
Apply suggestions from code review
wemeetagain Jan 28, 2026
15aef31
chore: add better logs
wemeetagain Jan 28, 2026
23aac57
chore: better error messages
wemeetagain Jan 29, 2026
f359c4f
chore: simplify data column sidecar getters interfaces
wemeetagain Jan 29, 2026
5c6e69e
Merge branch 'unstable' into cayman/unfinalized-block-write-queue
wemeetagain Jan 29, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/beacon-node/src/api/impl/beacon/blocks/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ export function getBeaconBlockApi({
() =>
// there is no rush to persist block since we published it to gossip anyway
chain
.processBlock(blockForImport, {...opts, eagerPersistBlock: false})
.processBlock(blockForImport, opts)
.catch((e) => {
if (e instanceof BlockError && e.type.code === BlockErrorCode.PARENT_UNKNOWN) {
chain.emitter.emit(ChainEvent.unknownParent, {
Expand Down
8 changes: 2 additions & 6 deletions packages/beacon-node/src/chain/blocks/importBlock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ import {toCheckpointHex} from "../stateCache/index.js";
import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js";
import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
import {getCheckpointFromState} from "./utils/checkpoint.js";
import {writeBlockInputToDb} from "./writeBlockInputToDb.js";

/**
* Fork-choice allows to import attestations from current (0) or past (1) epoch.
Expand Down Expand Up @@ -91,11 +90,8 @@ export async function importBlock(
throw Error("Unavailable block can not be imported in forkchoice");
}

// 1. Persist block to hot DB (pre-emptively)
// If eagerPersistBlock = true we do that in verifyBlocksInEpoch to batch all I/O operations to save block time to head
if (!opts.eagerPersistBlock) {
await writeBlockInputToDb.call(this, [blockInput]);
}
// 1. Persist block to hot DB (performed asynchronously to avoid blocking head selection)
await this.unfinalizedBlockWrites.push([blockInput]);
Comment thread
wemeetagain marked this conversation as resolved.
Resolved

// Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable
this.serializedCache.clear();
Expand Down
19 changes: 0 additions & 19 deletions packages/beacon-node/src/chain/blocks/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import {FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
import {assertLinearChainSegment} from "./utils/chainSegment.js";
import {verifyBlocksInEpoch} from "./verifyBlock.js";
import {verifyBlocksSanityChecks} from "./verifyBlocksSanityChecks.js";
import {removeEagerlyPersistedBlockInputs} from "./writeBlockInputToDb.js";

export {AttestationImportOpt, type ImportBlockOpts} from "./types.js";

Expand Down Expand Up @@ -143,24 +142,6 @@ export async function processBlocks(
}
}

// Clean db if we don't have blocks in forkchoice but already persisted them to db
//
// NOTE: this function is awaited to ensure that DB size remains constant, otherwise an attacker may bloat the
// disk with big malicious payloads. Our sequential block importer will wait for this promise before importing
// another block. The removal call error is not propagated since that would halt the chain.
//
// LOG: Because the error is not propagated and there's a risk of db bloat, the error is logged at warn level
// to alert the user of potential db bloat. This error _should_ never happen user must act and report to us
if (opts.eagerPersistBlock) {
await removeEagerlyPersistedBlockInputs.call(this, blocks).catch((e) => {
this.logger.warn(
"Error pruning eagerly imported block inputs, DB may grow in size if this error happens frequently",
{slot: blocks.map((block) => block.getBlock().message.slot).join(",")},
e
);
});
}
Comment thread
wemeetagain marked this conversation as resolved.
Resolved

throw err;
}
}
Expand Down
2 changes: 0 additions & 2 deletions packages/beacon-node/src/chain/blocks/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,6 @@ export type ImportBlockOpts = {
validBlobSidecars?: BlobSidecarValidation;
/** Seen timestamp seconds */
seenTimestampSec?: number;
/** Set to true if persist block right at verification time */
eagerPersistBlock?: boolean;
};

/**
Expand Down
8 changes: 0 additions & 8 deletions packages/beacon-node/src/chain/blocks/verifyBlock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import {verifyBlocksDataAvailability} from "./verifyBlocksDataAvailability.js";
import {SegmentExecStatus, verifyBlocksExecutionPayload} from "./verifyBlocksExecutionPayloads.js";
import {verifyBlocksSignatures} from "./verifyBlocksSignatures.js";
import {verifyBlocksStateTransitionOnly} from "./verifyBlocksStateTransitionOnly.js";
import {writeBlockInputToDb} from "./writeBlockInputToDb.js";

/**
* Verifies 1 or more blocks are fully valid; from a linear sequence of blocks.
Expand Down Expand Up @@ -150,13 +149,6 @@ export async function verifyBlocksInEpoch(
opts
)
: Promise.resolve({verifySignaturesTime: Date.now()}),

// ideally we want to only persist blocks after verifying them however the reality is there are
// rarely invalid blocks we'll batch all I/O operation here to reduce the overhead if there's
// an error, we'll remove blocks not in forkchoice
opts.verifyOnly !== true && opts.eagerPersistBlock
? writeBlockInputToDb.call(this, blockInputs)
: Promise.resolve(),
]);

if (opts.verifyOnly !== true) {
Expand Down
35 changes: 5 additions & 30 deletions packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,35 +98,10 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInputs: IBloc
}
}

/**
* Prunes eagerly persisted block inputs only if not known to the fork-choice
*/
export async function removeEagerlyPersistedBlockInputs(this: BeaconChain, blockInputs: IBlockInput[]): Promise<void> {
const blockToRemove = [];
const blobsToRemove = [];
const dataColumnsToRemove = [];

for (const blockInput of blockInputs) {
const block = blockInput.getBlock();
const slot = block.message.slot;
const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message);
const blockRootHex = toRootHex(blockRoot);
if (!this.forkChoice.hasBlockHex(blockRootHex)) {
blockToRemove.push(block);

if (isBlockInputColumns(blockInput) && blockInput.getCustodyColumns().length > 0) {
dataColumnsToRemove.push(blockRoot);
} else if (isBlockInputBlobs(blockInput)) {
const blobSidecars = blockInput.getBlobs();
blobsToRemove.push({blockRoot, slot, blobSidecars});
}
export async function persistBlockInputs(this: BeaconChain, blockInputs: IBlockInput[]): Promise<void> {
Comment thread
wemeetagain marked this conversation as resolved.
Resolved
await writeBlockInputToDb.call(this, blockInputs).finally(() => {
Comment thread
wemeetagain marked this conversation as resolved.
Resolved
for (const blockInput of blockInputs) {
this.seenBlockInputCache.prune(blockInput.blockRootHex);
}
}

await Promise.all([
// TODO: Batch DB operations not with Promise.all but with level db ops
this.db.block.batchRemove(blockToRemove),
this.db.blobSidecars.batchRemove(blobsToRemove),
this.db.dataColumnSidecar.deleteMany(dataColumnsToRemove),
]);
});
}
27 changes: 27 additions & 0 deletions packages/beacon-node/src/chain/chain.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,14 @@ import {CustodyConfig, getValidatorsCustodyRequirement} from "../util/dataColumn
import {callInNextEventLoop} from "../util/eventLoop.js";
import {ensureDir, writeIfNotExist} from "../util/file.js";
import {isOptimisticBlock} from "../util/forkChoice.js";
import {JobItemQueue} from "../util/queue/itemQueue.ts";
import {SerializedCache} from "../util/serializedCache.js";
import {ArchiveStore} from "./archiveStore/archiveStore.js";
import {CheckpointBalancesCache} from "./balancesCache.js";
import {BeaconProposerCache} from "./beaconProposerCache.js";
import {IBlockInput} from "./blocks/blockInput/index.js";
import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js";
import {persistBlockInputs} from "./blocks/writeBlockInputToDb.ts";
import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier, IBlsVerifier} from "./bls/index.js";
import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js";
import {ChainEvent, ChainEventEmitter} from "./emitter.js";
Expand Down Expand Up @@ -113,6 +115,11 @@ import {ValidatorMonitor} from "./validatorMonitor.js";
*/
const DEFAULT_MAX_CACHED_PRODUCED_RESULTS = 4;

/**
* The maximum number of pending unfinalized block writes to the database before backpressure is applied.
*/
const DEFAULT_MAX_PENDING_UNFINALIZED_BLOCK_WRITES = 32;

export class BeaconChain implements IBeaconChain {
readonly genesisTime: UintNum64;
readonly genesisValidatorsRoot: Root;
Expand All @@ -136,6 +143,7 @@ export class BeaconChain implements IBeaconChain {
readonly lightClientServer?: LightClientServer;
readonly reprocessController: ReprocessController;
readonly archiveStore: ArchiveStore;
readonly unfinalizedBlockWrites: JobItemQueue<[IBlockInput[]], void>;

// Ops pool
readonly attestationPool: AttestationPool;
Expand Down Expand Up @@ -405,6 +413,11 @@ export class BeaconChain implements IBeaconChain {
signal
);

this.unfinalizedBlockWrites = new JobItemQueue(persistBlockInputs.bind(this), {
Comment thread
wemeetagain marked this conversation as resolved.
Resolved
maxLength: DEFAULT_MAX_PENDING_UNFINALIZED_BLOCK_WRITES,
signal,
});

// always run PrepareNextSlotScheduler except for fork_choice spec tests
if (!opts?.disablePrepareNextSlot) {
new PrepareNextSlotScheduler(this, this.config, metrics, this.logger, signal);
Expand Down Expand Up @@ -652,6 +665,13 @@ export class BeaconChain implements IBeaconChain {
// Unfinalized slot, attempt to find in fork-choice
const block = this.forkChoice.getCanonicalBlockAtSlot(slot);
if (block) {
// Block found in fork-choice.
// It may be in the block input cache, awaiting full DA reconstruction, check there first
// Otherwise (most likely), check the hot db
const blockInput = this.seenBlockInputCache.get(block.blockRoot);
if (blockInput?.hasBlock()) {
return {block: blockInput.getBlock(), executionOptimistic: isOptimisticBlock(block), finalized: false};
}
const data = await this.db.block.get(fromHex(block.blockRoot));
if (data) {
return {block: data, executionOptimistic: isOptimisticBlock(block), finalized: false};
Expand All @@ -671,6 +691,13 @@ export class BeaconChain implements IBeaconChain {
): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean} | null> {
const block = this.forkChoice.getBlockHex(root);
if (block) {
// Block found in fork-choice.
// It may be in the block input cache, awaiting full DA reconstruction, check there first
// Otherwise (most likely), check the hot db
const blockInput = this.seenBlockInputCache.get(block.blockRoot);
if (blockInput?.hasBlock()) {
return {block: blockInput.getBlock(), executionOptimistic: isOptimisticBlock(block), finalized: false};
}
const data = await this.db.block.get(fromHex(root));
if (data) {
return {block: data, executionOptimistic: isOptimisticBlock(block), finalized: false};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -436,14 +436,11 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
blsVerifyOnMainThread: true,
// to track block process steps
seenTimestampSec,
// gossip block is validated, we want to process it asap
eagerPersistBlock: true,
})
.then(() => {
// Returns the delay between the start of `block.slot` and `current time`
const delaySec = chain.clock.secFromSlot(slot);
metrics?.gossipBlock.elapsedTimeTillProcessed.observe(delaySec);
chain.seenBlockInputCache.prune(blockInput.blockRootHex);
})
.catch((e) => {
// Adjust verbosity based on error type
Expand Down
3 changes: 0 additions & 3 deletions packages/beacon-node/src/sync/range/range.ts
Original file line number Diff line number Diff line change
Expand Up @@ -188,9 +188,6 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
// when this runs, syncing is the most important thing and gossip is not likely to run
// so we can utilize worker threads to verify signatures
blsVerifyOnMainThread: false,
// we want to be safe to only persist blocks after verifying it to avoid any attacks that may cause our DB
// to grow too much
eagerPersistBlock: false,
};

if (this.opts?.disableProcessAsChainSegment) {
Expand Down
2 changes: 0 additions & 2 deletions packages/beacon-node/src/sync/unknownBlock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -423,8 +423,6 @@ export class BlockInputSync {
// see https://github.com/ChainSafe/lodestar/issues/5650
ignoreIfFinalized: true,
blsVerifyOnMainThread: true,
// block is validated with correct root, we want to process it as soon as possible
eagerPersistBlock: true,
})
);

Expand Down
Loading