diff --git a/packages/api/src/beacon/routes/lodestar.ts b/packages/api/src/beacon/routes/lodestar.ts index 44771b432176..85c79d825933 100644 --- a/packages/api/src/beacon/routes/lodestar.ts +++ b/packages/api/src/beacon/routes/lodestar.ts @@ -81,7 +81,7 @@ export type Api = { /** TODO: description */ getSyncChainsDebugState(): Promise>; /** Dump all items in a gossip queue, by gossipType */ - getGossipQueueItems(gossipType: string): Promise>; + getGossipQueueItems(gossipType: string): Promise>; /** Dump all items in the regen queue */ getRegenQueueItems(): Promise>; /** Dump all items in the block processor queue */ diff --git a/packages/beacon-node/src/api/impl/beacon/pool/index.ts b/packages/beacon-node/src/api/impl/beacon/pool/index.ts index 9ee92af4a7a1..cb0f1893b295 100644 --- a/packages/beacon-node/src/api/impl/beacon/pool/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/pool/index.ts @@ -9,7 +9,7 @@ import {validateBlsToExecutionChange} from "../../../../chain/validation/blsToEx import {validateSyncCommitteeSigOnly} from "../../../../chain/validation/syncCommittee.js"; import {ApiModules} from "../../types.js"; import {AttestationError, GossipAction, SyncCommitteeError} from "../../../../chain/errors/index.js"; -import {validateGossipFnRetryUnknownRoot} from "../../../../network/gossip/handlers/index.js"; +import {validateGossipFnRetryUnknownRoot} from "../../../../network/processor/gossipHandlers.js"; export function getBeaconPoolApi({ chain, @@ -53,12 +53,12 @@ export function getBeaconPoolApi({ attestations.map(async (attestation, i) => { try { // eslint-disable-next-line @typescript-eslint/explicit-function-return-type - const validateFn = () => validateGossipAttestation(chain, attestation, null); + const validateFn = () => validateGossipAttestation(chain, attestation, null, null); const {slot, beaconBlockRoot} = attestation.data; // when a validator is configured with multiple beacon node urls, this attestation data may come from another beacon node // and the block hasn't been in our forkchoice since we haven't seen / processing that block // see https://github.com/ChainSafe/lodestar/issues/5098 - const {indexedAttestation, subnet} = await validateGossipFnRetryUnknownRoot( + const {indexedAttestation, subnet, attDataRootHex} = await validateGossipFnRetryUnknownRoot( validateFn, chain, slot, @@ -66,7 +66,7 @@ export function getBeaconPoolApi({ ); if (network.attnetsService.shouldProcess(subnet, slot)) { - const insertOutcome = chain.attestationPool.add(attestation); + const insertOutcome = chain.attestationPool.add(attestation, attDataRootHex); metrics?.opPool.attestationPoolInsertOutcome.inc({insertOutcome}); } const sentPeers = await network.gossip.publishBeaconAttestation(attestation, subnet); diff --git a/packages/beacon-node/src/api/impl/lodestar/index.ts b/packages/beacon-node/src/api/impl/lodestar/index.ts index d779f7d18cdc..7e69053a5841 100644 --- a/packages/beacon-node/src/api/impl/lodestar/index.ts +++ b/packages/beacon-node/src/api/impl/lodestar/index.ts @@ -60,7 +60,7 @@ export function getLodestarApi({ async getGossipQueueItems(gossipType: GossipType | string) { return { - data: await network.dumpGossipQueueItems(gossipType), + data: await network.dumpGossipQueue(gossipType as GossipType), }; }, diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index fa7c27864e49..2c2891d20064 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -25,7 +25,7 @@ import {CommitteeSubscription} from "../../../network/subnets/index.js"; import {ApiModules} from "../types.js"; import {RegenCaller} from "../../../chain/regen/index.js"; import {getValidatorStatus} from "../beacon/state/utils.js"; -import {validateGossipFnRetryUnknownRoot} from "../../../network/gossip/handlers/index.js"; +import {validateGossipFnRetryUnknownRoot} from "../../../network/processor/gossipHandlers.js"; import {computeSubnetForCommitteesAtSlot, getPubkeysForIndices} from "./utils.js"; /** @@ -541,7 +541,7 @@ export function getValidatorApi({ // when a validator is configured with multiple beacon node urls, this attestation may come from another beacon node // and the block hasn't been in our forkchoice since we haven't seen / processing that block // see https://github.com/ChainSafe/lodestar/issues/5098 - const {indexedAttestation, committeeIndices} = await validateGossipFnRetryUnknownRoot( + const {indexedAttestation, committeeIndices, attDataRootHex} = await validateGossipFnRetryUnknownRoot( validateFn, chain, slot, @@ -550,6 +550,7 @@ export function getValidatorApi({ chain.aggregatedAttestationPool.add( signedAggregateAndProof.message.aggregate, + attDataRootHex, indexedAttestation.attestingIndices.length, committeeIndices ); diff --git a/packages/beacon-node/src/chain/bls/interface.ts b/packages/beacon-node/src/chain/bls/interface.ts index 4ce95a675d01..2abeca62e103 100644 --- a/packages/beacon-node/src/chain/bls/interface.ts +++ b/packages/beacon-node/src/chain/bls/interface.ts @@ -43,4 +43,9 @@ export interface IBlsVerifier { /** For multithread pool awaits terminating all workers */ close(): Promise; + + /** + * Returns true if BLS worker pool is ready to accept more work jobs. + */ + canAcceptWork(): boolean; } diff --git a/packages/beacon-node/src/chain/bls/multithread/index.ts b/packages/beacon-node/src/chain/bls/multithread/index.ts index bc67ff0a2e1f..5c2ba9bd51fd 100644 --- a/packages/beacon-node/src/chain/bls/multithread/index.ts +++ b/packages/beacon-node/src/chain/bls/multithread/index.ts @@ -56,6 +56,11 @@ const MAX_BUFFERED_SIGS = 32; */ const MAX_BUFFER_WAIT_MS = 100; +/** + * Max concurrent jobs on `canAcceptWork` status + */ +const MAX_JOBS_CAN_ACCEPT_WORK = 512; + type WorkerApi = { verifyManySignatureSets(workReqArr: BlsWorkReq[]): Promise; }; @@ -110,6 +115,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { } | null = null; private blsVerifyAllMultiThread: boolean; private closed = false; + private workersBusy = 0; constructor(options: BlsMultiThreadWorkerPoolOptions, modules: BlsMultiThreadWorkerPoolModules) { const {logger, metrics} = modules; @@ -127,10 +133,21 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { this.workers = this.createWorkers(implementation, defaultPoolSize); if (metrics) { - metrics.blsThreadPool.queueLength.addCollect(() => metrics.blsThreadPool.queueLength.set(this.jobs.length)); + metrics.blsThreadPool.queueLength.addCollect(() => { + metrics.blsThreadPool.queueLength.set(this.jobs.length); + metrics.blsThreadPool.workersBusy.set(this.workersBusy); + }); } } + canAcceptWork(): boolean { + return ( + this.workersBusy < defaultPoolSize && + // TODO: Should also bound the jobs queue? + this.jobs.length < MAX_JOBS_CAN_ACCEPT_WORK + ); + } + async verifySignatureSets(sets: ISignatureSet[], opts: VerifySignatureOpts = {}): Promise { // Pubkeys are aggregated in the main thread regardless if verified in workers or in main thread this.metrics?.bls.aggregatedPubkeys.inc(getAggregatedPubkeysCount(sets)); @@ -310,6 +327,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { const workerApi = worker.status.workerApi; worker.status = {code: WorkerStatusCode.running, workerApi}; + this.workersBusy++; try { let startedSigSets = 0; @@ -375,6 +393,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { } worker.status = {code: WorkerStatusCode.idle, workerApi}; + this.workersBusy--; // Potentially run a new job setTimeout(this.runJob, 0); diff --git a/packages/beacon-node/src/chain/bls/singleThread.ts b/packages/beacon-node/src/chain/bls/singleThread.ts index 6895e2225696..78f3f4bf5200 100644 --- a/packages/beacon-node/src/chain/bls/singleThread.ts +++ b/packages/beacon-node/src/chain/bls/singleThread.ts @@ -37,4 +37,9 @@ export class BlsSingleThreadVerifier implements IBlsVerifier { async close(): Promise { // nothing to do } + + canAcceptWork(): boolean { + // Since sigs are verified blocking the main thread, there's no mechanism to throttle + return true; + } } diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 7c9c333e897b..f08da9c7add3 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -34,7 +34,7 @@ import {BeaconClock, LocalClock} from "./clock/index.js"; import {ChainEventEmitter, ChainEvent} from "./emitter.js"; import {IBeaconChain, ProposerPreparationData} from "./interface.js"; import {IChainOptions} from "./options.js"; -import {IStateRegenerator, QueuedStateRegenerator, RegenCaller} from "./regen/index.js"; +import {QueuedStateRegenerator, RegenCaller} from "./regen/index.js"; import {initializeForkChoice} from "./forkChoice/index.js"; import {computeAnchorCheckpoint} from "./initState.js"; import {IBlsVerifier, BlsSingleThreadVerifier, BlsMultiThreadWorkerPool} from "./bls/index.js"; @@ -64,6 +64,7 @@ import {AssembledBlockType, BlobsResultType, BlockType} from "./produceBlock/ind import {BlockAttributes, produceBlockBody} from "./produceBlock/produceBlockBody.js"; import {computeNewStateRoot} from "./produceBlock/computeNewStateRoot.js"; import {BlockInput} from "./blocks/types.js"; +import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js"; /** * Arbitrary constants, blobs should be consumed immediately in the same slot they are produced. @@ -91,7 +92,7 @@ export class BeaconChain implements IBeaconChain { readonly emitter: ChainEventEmitter; readonly stateCache: StateContextCache; readonly checkpointStateCache: CheckpointStateCache; - readonly regen: IStateRegenerator; + readonly regen: QueuedStateRegenerator; readonly lightClientServer: LightClientServer; readonly reprocessController: ReprocessController; @@ -109,6 +110,7 @@ export class BeaconChain implements IBeaconChain { readonly seenBlockProposers = new SeenBlockProposers(); readonly seenSyncCommitteeMessages = new SeenSyncCommitteeMessages(); readonly seenContributionAndProof: SeenContributionAndProof; + readonly seenAttestationDatas: SeenAttestationDatas; // Seen cache for liveness checks readonly seenBlockAttesters = new SeenBlockAttesters(); @@ -195,6 +197,7 @@ export class BeaconChain implements IBeaconChain { this.seenAggregatedAttestations = new SeenAggregatedAttestations(metrics); this.seenContributionAndProof = new SeenContributionAndProof(metrics); + this.seenAttestationDatas = new SeenAttestationDatas(metrics, this.opts?.attDataCacheSlotDistance); this.beaconProposerCache = new BeaconProposerCache(opts); this.checkpointBalancesCache = new CheckpointBalancesCache(); @@ -281,6 +284,14 @@ export class BeaconChain implements IBeaconChain { await this.bls.close(); } + regenCanAcceptWork(): boolean { + return this.regen.canAcceptWork(); + } + + blsThreadPoolCanAcceptWork(): boolean { + return this.bls.canAcceptWork(); + } + validatorSeenAtEpoch(index: ValidatorIndex, epoch: Epoch): boolean { // Caller must check that epoch is not older that current epoch - 1 // else the caches for that epoch may already be pruned. @@ -632,6 +643,7 @@ export class BeaconChain implements IBeaconChain { this.aggregatedAttestationPool.prune(slot); this.syncCommitteeMessagePool.prune(slot); this.seenSyncCommitteeMessages.prune(slot); + this.seenAttestationDatas.onSlot(slot); this.reprocessController.onSlot(slot); if (isFinite(this.config.BELLATRIX_FORK_EPOCH) && slot % this.exchangeTransitionConfigurationEverySlots === 0) { diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index d087d2b2a3ce..80ef3a2cf938 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -1,5 +1,5 @@ import {allForks, UintNum64, Root, phase0, Slot, RootHex, Epoch, ValidatorIndex, deneb, Wei} from "@lodestar/types"; -import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; +import {CachedBeaconStateAllForks, Index2PubkeyCache, PubkeyIndexMap} from "@lodestar/state-transition"; import {BeaconConfig} from "@lodestar/config"; import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import {Logger} from "@lodestar/utils"; @@ -31,6 +31,7 @@ import {SeenBlockAttesters} from "./seenCache/seenBlockAttesters.js"; import {CheckpointBalancesCache} from "./balancesCache.js"; import {IChainOptions} from "./options.js"; import {AssembledBlockType, BlockAttributes, BlockType} from "./produceBlock/produceBlockBody.js"; +import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js"; export type Eth2Context = { activeValidatorCount: number; @@ -68,6 +69,8 @@ export interface IBeaconChain { readonly regen: IStateRegenerator; readonly lightClientServer: LightClientServer; readonly reprocessController: ReprocessController; + readonly pubkey2index: PubkeyIndexMap; + readonly index2pubkey: Index2PubkeyCache; // Ops pool readonly attestationPool: AttestationPool; @@ -83,6 +86,7 @@ export interface IBeaconChain { readonly seenBlockProposers: SeenBlockProposers; readonly seenSyncCommitteeMessages: SeenSyncCommitteeMessages; readonly seenContributionAndProof: SeenContributionAndProof; + readonly seenAttestationDatas: SeenAttestationDatas; // Seen cache for liveness checks readonly seenBlockAttesters: SeenBlockAttesters; @@ -133,6 +137,9 @@ export interface IBeaconChain { /** Persist bad items to persistInvalidSszObjectsDir dir, for example invalid state, attestations etc. */ persistInvalidSszView(view: TreeView, suffix?: string): void; updateBuilderStatus(clockSlot: Slot): void; + + regenCanAcceptWork(): boolean; + blsThreadPoolCanAcceptWork(): boolean; } export type SSZObjectType = diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index 2b7711261865..5432fc62cfd7 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -6,7 +6,7 @@ import { SLOTS_PER_EPOCH, TIMELY_SOURCE_FLAG_INDEX, } from "@lodestar/params"; -import {phase0, Epoch, Slot, ssz, ValidatorIndex} from "@lodestar/types"; +import {phase0, Epoch, Slot, ssz, ValidatorIndex, RootHex} from "@lodestar/types"; import { CachedBeaconStateAllForks, CachedBeaconStatePhase0, @@ -70,7 +70,12 @@ export class AggregatedAttestationPool { return {attestationCount, attestationDataCount}; } - add(attestation: phase0.Attestation, attestingIndicesCount: number, committee: ValidatorIndex[]): InsertOutcome { + add( + attestation: phase0.Attestation, + dataRootHex: RootHex, + attestingIndicesCount: number, + committee: ValidatorIndex[] + ): InsertOutcome { const slot = attestation.data.slot; const lowestPermissibleSlot = this.lowestPermissibleSlot; @@ -80,9 +85,6 @@ export class AggregatedAttestationPool { } const attestationGroupByDataHash = this.attestationGroupByDataHashBySlot.getOrDefault(slot); - const dataRoot = ssz.phase0.AttestationData.hashTreeRoot(attestation.data); - const dataRootHex = toHexString(dataRoot); - let attestationGroup = attestationGroupByDataHash.get(dataRootHex); if (!attestationGroup) { attestationGroup = new MatchingDataAttestationGroup(committee, attestation.data); diff --git a/packages/beacon-node/src/chain/opPools/attestationPool.ts b/packages/beacon-node/src/chain/opPools/attestationPool.ts index 59e86d8f4e7d..8494fc538d28 100644 --- a/packages/beacon-node/src/chain/opPools/attestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/attestationPool.ts @@ -1,4 +1,4 @@ -import {phase0, Slot, Root, ssz} from "@lodestar/types"; +import {phase0, Slot, Root, RootHex} from "@lodestar/types"; import {PointFormat, Signature} from "@chainsafe/bls/types"; import bls from "@chainsafe/bls"; import {BitArray, toHexString} from "@chainsafe/ssz"; @@ -93,7 +93,7 @@ export class AttestationPool { * - Valid committeeIndex * - Valid data */ - add(attestation: phase0.Attestation): InsertOutcome { + add(attestation: phase0.Attestation, attDataRootHex: RootHex): InsertOutcome { const slot = attestation.data.slot; const lowestPermissibleSlot = this.lowestPermissibleSlot; @@ -113,17 +113,14 @@ export class AttestationPool { throw new OpPoolError({code: OpPoolErrorCode.REACHED_MAX_PER_SLOT}); } - const dataRoot = ssz.phase0.AttestationData.hashTreeRoot(attestation.data); - const dataRootHex = toHexString(dataRoot); - // Pre-aggregate the contribution with existing items - const aggregate = aggregateByRoot.get(dataRootHex); + const aggregate = aggregateByRoot.get(attDataRootHex); if (aggregate) { // Aggregate mutating return aggregateAttestationInto(aggregate, attestation); } else { // Create new aggregate - aggregateByRoot.set(dataRootHex, attestationToAggregate(attestation)); + aggregateByRoot.set(attDataRootHex, attestationToAggregate(attestation)); return InsertOutcome.NewData; } } diff --git a/packages/beacon-node/src/chain/opPools/types.ts b/packages/beacon-node/src/chain/opPools/types.ts index e91ec377178d..62a99b805935 100644 --- a/packages/beacon-node/src/chain/opPools/types.ts +++ b/packages/beacon-node/src/chain/opPools/types.ts @@ -11,6 +11,8 @@ export enum InsertOutcome { AlreadyKnown = "AlreadyKnown", /** Not existing in the pool but it's too old to add. No changes were made. */ Old = "Old", + /** The pool has reached its limit. No changes were made. */ + ReachLimit = "ReachLimit", /** Attestation comes to the pool at > 2/3 of slot. No changes were made */ Late = "Late", /** The data is know, and the new participants have been added to the aggregated signature */ diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index cd8b01113494..a4763a26f36c 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -6,6 +6,7 @@ import {LightClientServerOpts} from "./lightClient/index.js"; export type IChainOptions = BlockProcessOpts & PoolOpts & + SeenCacheOpts & ForkChoiceOpts & ArchiverOpts & LightClientServerOpts & { @@ -55,6 +56,13 @@ export type PoolOpts = { preaggregateSlotDistance?: number; }; +export type SeenCacheOpts = { + /** + * Slot distance from current slot to cache AttestationData + */ + attDataCacheSlotDistance?: number; +}; + export const defaultChainOptions: IChainOptions = { blsVerifyAllMainThread: false, blsVerifyAllMultiThread: false, diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index 16e2afaa33ea..b4ee806cfba4 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -10,6 +10,8 @@ import {StateRegenerator, RegenModules} from "./regen.js"; import {RegenError, RegenErrorCode} from "./errors.js"; const REGEN_QUEUE_MAX_LEN = 256; +// TODO: Should this constant be lower than above? 256 feels high +const REGEN_CAN_ACCEPT_WORK_THRESHOLD = 16; type QueuedStateRegeneratorModules = RegenModules & { signal: AbortSignal; @@ -46,6 +48,10 @@ export class QueuedStateRegenerator implements IStateRegenerator { this.metrics = modules.metrics; } + canAcceptWork(): boolean { + return this.jobQueue.jobLen < REGEN_CAN_ACCEPT_WORK_THRESHOLD; + } + /** * Get the state to run with `block`. * - State after `block.parentRoot` dialed forward to block.slot diff --git a/packages/beacon-node/src/chain/reprocess.ts b/packages/beacon-node/src/chain/reprocess.ts index 013b05bf0a9a..3ab6056fb3af 100644 --- a/packages/beacon-node/src/chain/reprocess.ts +++ b/packages/beacon-node/src/chain/reprocess.ts @@ -61,10 +61,10 @@ export class ReprocessController { * @returns true if blockFound */ waitForBlockOfAttestation(slot: Slot, root: RootHex): Promise { - this.metrics?.reprocessAttestations.total.inc(); + this.metrics?.reprocessApiAttestations.total.inc(); if (this.awaitingPromisesCount >= MAXIMUM_QUEUED_ATTESTATIONS) { - this.metrics?.reprocessAttestations.reject.inc({reason: ReprocessStatus.reached_limit}); + this.metrics?.reprocessApiAttestations.reject.inc({reason: ReprocessStatus.reached_limit}); return Promise.resolve(false); } @@ -116,8 +116,8 @@ export class ReprocessController { const {resolve, addedTimeMs, awaitingAttestationsCount} = awaitingPromise; resolve(true); this.awaitingPromisesCount -= awaitingAttestationsCount; - this.metrics?.reprocessAttestations.resolve.inc(awaitingAttestationsCount); - this.metrics?.reprocessAttestations.waitTimeBeforeResolve.set((Date.now() - addedTimeMs) / 1000); + this.metrics?.reprocessApiAttestations.resolve.inc(awaitingAttestationsCount); + this.metrics?.reprocessApiAttestations.waitSecBeforeResolve.set((Date.now() - addedTimeMs) / 1000); } // prune @@ -140,8 +140,8 @@ export class ReprocessController { for (const awaitingPromise of awaitingPromisesByRoot.values()) { const {resolve, addedTimeMs} = awaitingPromise; resolve(false); - this.metrics?.reprocessAttestations.waitTimeBeforeReject.set((now - addedTimeMs) / 1000); - this.metrics?.reprocessAttestations.reject.inc({reason: ReprocessStatus.expired}); + this.metrics?.reprocessApiAttestations.waitSecBeforeReject.set((now - addedTimeMs) / 1000); + this.metrics?.reprocessApiAttestations.reject.inc({reason: ReprocessStatus.expired}); } // prune diff --git a/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts b/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts new file mode 100644 index 000000000000..a3e083f2b041 --- /dev/null +++ b/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts @@ -0,0 +1,106 @@ +import {RootHex, Slot} from "@lodestar/types"; +import {MapDef} from "@lodestar/utils"; +import {Metrics} from "../../metrics/metrics.js"; +import {AttDataBase64} from "../../util/sszBytes.js"; +import {InsertOutcome} from "../opPools/types.js"; + +export type AttestationDataCacheEntry = { + // part of shuffling data, so this does not take memory + committeeIndices: number[]; + // IndexedAttestationData signing root, 32 bytes + signingRoot: Uint8Array; + // to be consumed by forkchoice + attDataRootHex: RootHex; + subnet: number; +}; + +enum RejectReason { + // attestation data reaches MAX_CACHE_SIZE_PER_SLOT + reached_limit = "reached_limit", + // attestation data is too old + too_old = "too_old", + // attestation data is already known + already_known = "already_known", +} + +/** + * There are maximum 64 committees per slot, assuming 1 committee may have up to 3 different data due to some nodes + * are not up to date, we can have up to 192 different attestation data per slot. + */ +const DEFAULT_MAX_CACHE_SIZE_PER_SLOT = 200; + +/** + * It takes less than 300kb to cache 200 attestation data per slot, so we can cache 3 slots worth of attestation data. + */ +const DEFAULT_CACHE_SLOT_DISTANCE = 2; + +/** + * As of April 2023, validating gossip attestation takes ~12% of cpu time for a node subscribing to all subnets on mainnet. + * Having this cache help saves a lot of cpu time since most of the gossip attestations are on the same slot. + */ +export class SeenAttestationDatas { + private cacheEntryByAttDataBase64BySlot = new MapDef>( + () => new Map() + ); + private lowestPermissibleSlot = 0; + + constructor( + private readonly metrics: Metrics | null, + private readonly cacheSlotDistance = DEFAULT_CACHE_SLOT_DISTANCE, + // mainly for unit test + private readonly maxCacheSizePerSlot = DEFAULT_MAX_CACHE_SIZE_PER_SLOT + ) { + metrics?.seenCache.attestationData.totalSlot.addCollect(() => this.onScrapeLodestarMetrics(metrics)); + } + + // TODO: Move InsertOutcome type definition to a common place + add(slot: Slot, attDataBase64: AttDataBase64, cacheEntry: AttestationDataCacheEntry): InsertOutcome { + if (slot < this.lowestPermissibleSlot) { + this.metrics?.seenCache.attestationData.reject.inc({reason: RejectReason.too_old}); + return InsertOutcome.Old; + } + + const cacheEntryByAttDataBase64 = this.cacheEntryByAttDataBase64BySlot.getOrDefault(slot); + if (cacheEntryByAttDataBase64.has(attDataBase64)) { + this.metrics?.seenCache.attestationData.reject.inc({reason: RejectReason.already_known}); + return InsertOutcome.AlreadyKnown; + } + + if (cacheEntryByAttDataBase64.size >= this.maxCacheSizePerSlot) { + this.metrics?.seenCache.attestationData.reject.inc({reason: RejectReason.reached_limit}); + return InsertOutcome.ReachLimit; + } + + cacheEntryByAttDataBase64.set(attDataBase64, cacheEntry); + return InsertOutcome.NewData; + } + + get(slot: Slot, attDataBase64: AttDataBase64): AttestationDataCacheEntry | null { + const cacheEntryByAttDataBase64 = this.cacheEntryByAttDataBase64BySlot.get(slot); + const cacheEntry = cacheEntryByAttDataBase64?.get(attDataBase64); + if (cacheEntry) { + this.metrics?.seenCache.attestationData.hit.inc(); + } else { + this.metrics?.seenCache.attestationData.miss.inc(); + } + return cacheEntry ?? null; + } + + onSlot(clockSlot: Slot): void { + this.lowestPermissibleSlot = Math.max(clockSlot - this.cacheSlotDistance, 0); + for (const slot of this.cacheEntryByAttDataBase64BySlot.keys()) { + if (slot < this.lowestPermissibleSlot) { + this.cacheEntryByAttDataBase64BySlot.delete(slot); + } + } + } + + private onScrapeLodestarMetrics(metrics: Metrics): void { + metrics?.seenCache.attestationData.totalSlot.set(this.cacheEntryByAttDataBase64BySlot.size); + // only track current slot + const currentSlot = this.lowestPermissibleSlot + this.cacheSlotDistance; + metrics?.seenCache.attestationData.countPerSlot.set( + this.cacheEntryByAttDataBase64BySlot.get(currentSlot)?.size ?? 0 + ); + } +} diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index 291821bb7ea6..44a2e500e31f 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -1,21 +1,31 @@ import {toHexString} from "@chainsafe/ssz"; -import {phase0, ssz, ValidatorIndex} from "@lodestar/types"; +import {phase0, RootHex, ssz, ValidatorIndex} from "@lodestar/types"; import { computeEpochAtSlot, isAggregatorFromCommitteeLength, getIndexedAttestationSignatureSet, + ISignatureSet, + createAggregateSignatureSetFromComponents, } from "@lodestar/state-transition"; import {IBeaconChain} from ".."; import {AttestationError, AttestationErrorCode, GossipAction} from "../errors/index.js"; import {RegenCaller} from "../regen/index.js"; +import {getAttDataBase64FromSignedAggregateAndProofSerialized} from "../../util/sszBytes.js"; import {getSelectionProofSignatureSet, getAggregateAndProofSignatureSet} from "./signatureSets/index.js"; import {getCommitteeIndices, verifyHeadBlockAndTargetRoot, verifyPropagationSlotRange} from "./attestation.js"; +export type AggregateAndProofValidationResult = { + indexedAttestation: phase0.IndexedAttestation; + committeeIndices: ValidatorIndex[]; + attDataRootHex: RootHex; +}; + export async function validateGossipAggregateAndProof( chain: IBeaconChain, signedAggregateAndProof: phase0.SignedAggregateAndProof, - skipValidationKnownAttesters = false -): Promise<{indexedAttestation: phase0.IndexedAttestation; committeeIndices: ValidatorIndex[]}> { + skipValidationKnownAttesters = false, + serializedData: Uint8Array | null = null +): Promise { // Do checks in this order: // - do early checks (w/o indexed attestation) // - > obtain indexed attestation and committes per slot @@ -27,22 +37,27 @@ export async function validateGossipAggregateAndProof( const aggregate = aggregateAndProof.aggregate; const {aggregationBits} = aggregate; const attData = aggregate.data; - const attDataRoot = toHexString(ssz.phase0.AttestationData.hashTreeRoot(attData)); const attSlot = attData.slot; + + const attDataBase64 = serializedData ? getAttDataBase64FromSignedAggregateAndProofSerialized(serializedData) : null; + const cachedAttData = attDataBase64 ? chain.seenAttestationDatas.get(attSlot, attDataBase64) : null; + const attIndex = attData.index; const attEpoch = computeEpochAtSlot(attSlot); const attTarget = attData.target; const targetEpoch = attTarget.epoch; - // [REJECT] The attestation's epoch matches its target -- i.e. attestation.data.target.epoch == compute_epoch_at_slot(attestation.data.slot) - if (targetEpoch !== attEpoch) { - throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.BAD_TARGET_EPOCH}); - } + if (!cachedAttData) { + // [REJECT] The attestation's epoch matches its target -- i.e. attestation.data.target.epoch == compute_epoch_at_slot(attestation.data.slot) + if (targetEpoch !== attEpoch) { + throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.BAD_TARGET_EPOCH}); + } - // [IGNORE] aggregate.data.slot is within the last ATTESTATION_PROPAGATION_SLOT_RANGE slots (with a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) - // -- i.e. aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot - // (a client MAY queue future aggregates for processing at the appropriate slot). - verifyPropagationSlotRange(chain, attSlot); + // [IGNORE] aggregate.data.slot is within the last ATTESTATION_PROPAGATION_SLOT_RANGE slots (with a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) + // -- i.e. aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot + // (a client MAY queue future aggregates for processing at the appropriate slot). + verifyPropagationSlotRange(chain, attSlot); + } // [IGNORE] The aggregate is the first valid aggregate received for the aggregator with // index aggregate_and_proof.aggregator_index for the epoch aggregate.data.target.epoch. @@ -57,14 +72,17 @@ export async function validateGossipAggregateAndProof( // _[IGNORE]_ A valid aggregate attestation defined by `hash_tree_root(aggregate.data)` whose `aggregation_bits` // is a non-strict superset has _not_ already been seen. + const attDataRootHex = cachedAttData + ? cachedAttData.attDataRootHex + : toHexString(ssz.phase0.AttestationData.hashTreeRoot(attData)); if ( !skipValidationKnownAttesters && - chain.seenAggregatedAttestations.isKnown(targetEpoch, attDataRoot, aggregationBits) + chain.seenAggregatedAttestations.isKnown(targetEpoch, attDataRootHex, aggregationBits) ) { throw new AttestationError(GossipAction.IGNORE, { code: AttestationErrorCode.ATTESTERS_ALREADY_KNOWN, targetEpoch, - aggregateRoot: attDataRoot, + aggregateRoot: attDataRootHex, }); } @@ -88,7 +106,9 @@ export async function validateGossipAggregateAndProof( }); }); - const committeeIndices: number[] = getCommitteeIndices(attHeadState, attSlot, attIndex); + const committeeIndices: number[] = cachedAttData + ? cachedAttData.committeeIndices + : getCommitteeIndices(attHeadState, attSlot, attIndex); const attestingIndices = aggregate.aggregationBits.intersectValues(committeeIndices); const indexedAttestation: phase0.IndexedAttestation = { @@ -122,11 +142,24 @@ export async function validateGossipAggregateAndProof( // [REJECT] The aggregator signature, signed_aggregate_and_proof.signature, is valid. // [REJECT] The signature of aggregate is valid. const aggregator = attHeadState.epochCtx.index2pubkey[aggregateAndProof.aggregatorIndex]; + let indexedAttestationSignatureSet: ISignatureSet; + if (cachedAttData) { + const {signingRoot} = cachedAttData; + indexedAttestationSignatureSet = createAggregateSignatureSetFromComponents( + indexedAttestation.attestingIndices.map((i) => chain.index2pubkey[i]), + signingRoot, + indexedAttestation.signature + ); + } else { + indexedAttestationSignatureSet = getIndexedAttestationSignatureSet(attHeadState, indexedAttestation); + } const signatureSets = [ getSelectionProofSignatureSet(attHeadState, attSlot, aggregator, signedAggregateAndProof), getAggregateAndProofSignatureSet(attHeadState, attEpoch, aggregator, signedAggregateAndProof), - getIndexedAttestationSignatureSet(attHeadState, indexedAttestation), + indexedAttestationSignatureSet, ]; + // no need to write to SeenAttestationDatas + if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true}))) { throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.INVALID_SIGNATURE}); } @@ -145,10 +178,10 @@ export async function validateGossipAggregateAndProof( chain.seenAggregators.add(targetEpoch, aggregatorIndex); chain.seenAggregatedAttestations.add( targetEpoch, - attDataRoot, + attDataRootHex, {aggregationBits, trueBitCount: attestingIndices.length}, false ); - return {indexedAttestation, committeeIndices}; + return {indexedAttestation, committeeIndices, attDataRootHex}; } diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index b463dbdf9533..97e9947fda72 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -1,23 +1,34 @@ -import {phase0, Epoch, Root, Slot} from "@lodestar/types"; +import {phase0, Epoch, Root, Slot, RootHex, ssz} from "@lodestar/types"; import {ProtoBlock} from "@lodestar/fork-choice"; import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH} from "@lodestar/params"; import {toHexString} from "@chainsafe/ssz"; import { computeEpochAtSlot, CachedBeaconStateAllForks, - getIndexedAttestationSignatureSet, + ISignatureSet, + getAttestationDataSigningRoot, + createAggregateSignatureSetFromComponents, } from "@lodestar/state-transition"; import {IBeaconChain} from ".."; import {AttestationError, AttestationErrorCode, GossipAction} from "../errors/index.js"; import {MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC} from "../../constants/index.js"; import {RegenCaller} from "../regen/index.js"; +import {getAttDataBase64FromAttestationSerialized} from "../../util/sszBytes.js"; + +export type AttestationValidationResult = { + indexedAttestation: phase0.IndexedAttestation; + subnet: number; + attDataRootHex: RootHex; +}; export async function validateGossipAttestation( chain: IBeaconChain, attestation: phase0.Attestation, /** Optional, to allow verifying attestations through API with unknown subnet */ - subnet: number | null -): Promise<{indexedAttestation: phase0.IndexedAttestation; subnet: number}> { + subnet: number | null, + // available for gossip attestations, null for api attestations + serializedData: Uint8Array | null = null +): Promise { // Do checks in this order: // - do early checks (w/o indexed attestation) // - > obtain indexed attestation and committes per slot @@ -29,21 +40,27 @@ export async function validateGossipAttestation( // Run the checks that happen before an indexed attestation is constructed. const attData = attestation.data; const attSlot = attData.slot; + const attIndex = attData.index; const attEpoch = computeEpochAtSlot(attSlot); const attTarget = attData.target; const targetEpoch = attTarget.epoch; - // [REJECT] The attestation's epoch matches its target -- i.e. attestation.data.target.epoch == compute_epoch_at_slot(attestation.data.slot) - if (targetEpoch !== attEpoch) { - throw new AttestationError(GossipAction.REJECT, { - code: AttestationErrorCode.BAD_TARGET_EPOCH, - }); - } + const attDataBase64 = serializedData ? getAttDataBase64FromAttestationSerialized(serializedData) : null; + const cachedAttData = attDataBase64 ? chain.seenAttestationDatas.get(attSlot, attDataBase64) : null; + + if (!cachedAttData) { + // [REJECT] The attestation's epoch matches its target -- i.e. attestation.data.target.epoch == compute_epoch_at_slot(attestation.data.slot) + if (targetEpoch !== attEpoch) { + throw new AttestationError(GossipAction.REJECT, { + code: AttestationErrorCode.BAD_TARGET_EPOCH, + }); + } - // [IGNORE] attestation.data.slot is within the last ATTESTATION_PROPAGATION_SLOT_RANGE slots (within a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) - // -- i.e. attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot - // (a client MAY queue future attestations for processing at the appropriate slot). - verifyPropagationSlotRange(chain, attSlot); + // [IGNORE] attestation.data.slot is within the last ATTESTATION_PROPAGATION_SLOT_RANGE slots (within a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) + // -- i.e. attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot + // (a client MAY queue future attestations for processing at the appropriate slot). + verifyPropagationSlotRange(chain, attSlot); + } // [REJECT] The attestation is unaggregated -- that is, it has exactly one participating validator // (len([bit for bit in attestation.aggregation_bits if bit]) == 1, i.e. exactly 1 bit is set). @@ -56,48 +73,58 @@ export async function validateGossipAttestation( }); } - // Attestations must be for a known block. If the block is unknown, we simply drop the - // attestation and do not delay consideration for later. - // - // TODO (LH): Enforce a maximum skip distance for unaggregated attestations. - - // [IGNORE] The block being voted for (attestation.data.beacon_block_root) has been seen (via both gossip - // and non-gossip sources) (a client MAY queue attestations for processing once block is retrieved). - const attHeadBlock = verifyHeadBlockAndTargetRoot(chain, attData.beaconBlockRoot, attTarget.root, attEpoch); - - // [REJECT] The block being voted for (attestation.data.beacon_block_root) passes validation. - // > Altready check in `verifyHeadBlockAndTargetRoot()` - - // [IGNORE] The current finalized_checkpoint is an ancestor of the block defined by attestation.data.beacon_block_root - // -- i.e. get_ancestor(store, attestation.data.beacon_block_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) == store.finalized_checkpoint.root - // > Altready check in `verifyHeadBlockAndTargetRoot()` - - // [REJECT] The attestation's target block is an ancestor of the block named in the LMD vote - // --i.e. get_ancestor(store, attestation.data.beacon_block_root, compute_start_slot_at_epoch(attestation.data.target.epoch)) == attestation.data.target.root - // > Altready check in `verifyHeadBlockAndTargetRoot()` - - // Using the target checkpoint state here caused unstable memory issue - // See https://github.com/ChainSafe/lodestar/issues/4896 - // TODO: https://github.com/ChainSafe/lodestar/issues/4900 - const attHeadState = await chain.regen - .getState(attHeadBlock.stateRoot, RegenCaller.validateGossipAttestation) - .catch((e: Error) => { - throw new AttestationError(GossipAction.REJECT, { - code: AttestationErrorCode.MISSING_ATTESTATION_HEAD_STATE, - error: e as Error, + let committeeIndices: number[]; + let getSigningRoot: () => Uint8Array; + let expectedSubnet: number; + if (cachedAttData) { + committeeIndices = cachedAttData.committeeIndices; + getSigningRoot = () => cachedAttData.signingRoot; + expectedSubnet = cachedAttData.subnet; + } else { + // Attestations must be for a known block. If the block is unknown, we simply drop the + // attestation and do not delay consideration for later. + // + // TODO (LH): Enforce a maximum skip distance for unaggregated attestations. + + // [IGNORE] The block being voted for (attestation.data.beacon_block_root) has been seen (via both gossip + // and non-gossip sources) (a client MAY queue attestations for processing once block is retrieved). + const attHeadBlock = verifyHeadBlockAndTargetRoot(chain, attData.beaconBlockRoot, attTarget.root, attEpoch); + + // [REJECT] The block being voted for (attestation.data.beacon_block_root) passes validation. + // > Altready check in `verifyHeadBlockAndTargetRoot()` + + // [IGNORE] The current finalized_checkpoint is an ancestor of the block defined by attestation.data.beacon_block_root + // -- i.e. get_ancestor(store, attestation.data.beacon_block_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) == store.finalized_checkpoint.root + // > Altready check in `verifyHeadBlockAndTargetRoot()` + + // [REJECT] The attestation's target block is an ancestor of the block named in the LMD vote + // --i.e. get_ancestor(store, attestation.data.beacon_block_root, compute_start_slot_at_epoch(attestation.data.target.epoch)) == attestation.data.target.root + // > Altready check in `verifyHeadBlockAndTargetRoot()` + + // Using the target checkpoint state here caused unstable memory issue + // See https://github.com/ChainSafe/lodestar/issues/4896 + // TODO: https://github.com/ChainSafe/lodestar/issues/4900 + const attHeadState = await chain.regen + .getState(attHeadBlock.stateRoot, RegenCaller.validateGossipAttestation) + .catch((e: Error) => { + throw new AttestationError(GossipAction.REJECT, { + code: AttestationErrorCode.MISSING_ATTESTATION_HEAD_STATE, + error: e as Error, + }); }); - }); - // [REJECT] The committee index is within the expected range - // -- i.e. data.index < get_committee_count_per_slot(state, data.target.epoch) - const attIndex = attData.index; - const committeeIndices = getCommitteeIndices(attHeadState, attSlot, attIndex); + // [REJECT] The committee index is within the expected range + // -- i.e. data.index < get_committee_count_per_slot(state, data.target.epoch) + committeeIndices = getCommitteeIndices(attHeadState, attSlot, attIndex); + getSigningRoot = () => getAttestationDataSigningRoot(attHeadState, attData); + expectedSubnet = attHeadState.epochCtx.computeSubnetForSlot(attSlot, attIndex); + } const validatorIndex = committeeIndices[bitIndex]; // [REJECT] The number of aggregation bits matches the committee size // -- i.e. len(attestation.aggregation_bits) == len(get_beacon_committee(state, data.slot, data.index)). - // > TODO: Is this necessary? Lighthouse does not do this check + // > TODO: Is this necessary? Lighthouse does not do this check. if (aggregationBits.bitLen !== committeeIndices.length) { throw new AttestationError(GossipAction.REJECT, { code: AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS, @@ -113,7 +140,6 @@ export async function validateGossipAttestation( // -- i.e. compute_subnet_for_attestation(committees_per_slot, attestation.data.slot, attestation.data.index) == subnet_id, // where committees_per_slot = get_committee_count_per_slot(state, attestation.data.target.epoch), // which may be pre-computed along with the committee information for the signature check. - const expectedSubnet = attHeadState.epochCtx.computeSubnetForSlot(attSlot, attIndex); if (subnet !== null && subnet !== expectedSubnet) { throw new AttestationError(GossipAction.REJECT, { code: AttestationErrorCode.INVALID_SUBNET_ID, @@ -133,12 +159,38 @@ export async function validateGossipAttestation( } // [REJECT] The signature of attestation is valid. - const indexedAttestation: phase0.IndexedAttestation = { - attestingIndices: [validatorIndex], - data: attData, - signature: attestation.signature, - }; - const signatureSet = getIndexedAttestationSignatureSet(attHeadState, indexedAttestation); + const attestingIndices = [validatorIndex]; + let signatureSet: ISignatureSet; + let attDataRootHex: RootHex; + if (cachedAttData) { + // there could be up to 6% of cpu time to compute signing root if we don't clone the signature set + signatureSet = createAggregateSignatureSetFromComponents( + attestingIndices.map((i) => chain.index2pubkey[i]), + cachedAttData.signingRoot, + attestation.signature + ); + attDataRootHex = cachedAttData.attDataRootHex; + } else { + signatureSet = createAggregateSignatureSetFromComponents( + attestingIndices.map((i) => chain.index2pubkey[i]), + getSigningRoot(), + attestation.signature + ); + + // add cached attestation data before verifying signature + attDataRootHex = toHexString(ssz.phase0.AttestationData.hashTreeRoot(attData)); + if (attDataBase64) { + chain.seenAttestationDatas.add(attSlot, attDataBase64, { + committeeIndices, + signingRoot: signatureSet.signingRoot, + subnet: expectedSubnet, + // precompute this to be used in forkchoice + // root of AttestationData was already cached during getIndexedAttestationSignatureSet + attDataRootHex, + }); + } + } + if (!(await chain.bls.verifySignatureSets([signatureSet], {batchable: true}))) { throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.INVALID_SIGNATURE}); } @@ -158,7 +210,12 @@ export async function validateGossipAttestation( chain.seenAttesters.add(targetEpoch, validatorIndex); - return {indexedAttestation, subnet: expectedSubnet}; + const indexedAttestation: phase0.IndexedAttestation = { + attestingIndices, + data: attData, + signature: attestation.signature, + }; + return {indexedAttestation, subnet: expectedSubnet, attDataRootHex}; } /** diff --git a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts index 043b2bd4cb01..c31b210e0f6a 100644 --- a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts @@ -6,24 +6,29 @@ import { CachedBeaconStateAllForks, computeSigningRoot, computeStartSlotAtEpoch, + createSingleSignatureSetFromComponents, ISignatureSet, - SignatureSetType, } from "@lodestar/state-transition"; -export function getAggregateAndProofSignatureSet( +export function getAggregateAndProofSigningRoot( state: CachedBeaconStateAllForks, epoch: Epoch, - aggregator: PublicKey, aggregateAndProof: phase0.SignedAggregateAndProof -): ISignatureSet { +): Uint8Array { const slot = computeStartSlotAtEpoch(epoch); const aggregatorDomain = state.config.getDomain(state.slot, DOMAIN_AGGREGATE_AND_PROOF, slot); - const signingRoot = computeSigningRoot(ssz.phase0.AggregateAndProof, aggregateAndProof.message, aggregatorDomain); + return computeSigningRoot(ssz.phase0.AggregateAndProof, aggregateAndProof.message, aggregatorDomain); +} - return { - type: SignatureSetType.single, - pubkey: aggregator, - signingRoot, - signature: aggregateAndProof.signature, - }; +export function getAggregateAndProofSignatureSet( + state: CachedBeaconStateAllForks, + epoch: Epoch, + aggregator: PublicKey, + aggregateAndProof: phase0.SignedAggregateAndProof +): ISignatureSet { + return createSingleSignatureSetFromComponents( + aggregator, + getAggregateAndProofSigningRoot(state, epoch, aggregateAndProof), + aggregateAndProof.signature + ); } diff --git a/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts b/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts index f82b5b80af59..7c19091992e6 100644 --- a/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts +++ b/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts @@ -4,22 +4,24 @@ import type {PublicKey} from "@chainsafe/bls/types"; import { CachedBeaconStateAllForks, computeSigningRoot, + createSingleSignatureSetFromComponents, ISignatureSet, - SignatureSetType, } from "@lodestar/state-transition"; +export function getSelectionProofSigningRoot(state: CachedBeaconStateAllForks, slot: Slot): Uint8Array { + const selectionProofDomain = state.config.getDomain(state.slot, DOMAIN_SELECTION_PROOF, slot); + return computeSigningRoot(ssz.Slot, slot, selectionProofDomain); +} + export function getSelectionProofSignatureSet( state: CachedBeaconStateAllForks, slot: Slot, aggregator: PublicKey, aggregateAndProof: phase0.SignedAggregateAndProof ): ISignatureSet { - const selectionProofDomain = state.config.getDomain(state.slot, DOMAIN_SELECTION_PROOF, slot); - - return { - type: SignatureSetType.single, - pubkey: aggregator, - signingRoot: computeSigningRoot(ssz.Slot, slot, selectionProofDomain), - signature: aggregateAndProof.message.selectionProof, - }; + return createSingleSignatureSetFromComponents( + aggregator, + getSelectionProofSigningRoot(state, slot), + aggregateAndProof.message.selectionProof + ); } diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 83600bcc090b..c75dd7ab7747 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -267,10 +267,26 @@ export function createLodestarMetrics( }), gossipValidationQueueConcurrency: register.gauge<"topic">({ name: "lodestar_gossip_validation_queue_concurrency", - help: "Current concurrency of gossip validation queue", + help: "Current count of jobs being run on network processor for topic", labelNames: ["topic"], }), + networkProcessor: { + executeWorkCalls: register.gauge({ + name: "lodestar_network_processor_execute_work_calls_total", + help: "Total calls to network processor execute work fn", + }), + jobsSubmitted: register.histogram({ + name: "lodestar_network_processor_execute_jobs_submitted_total", + help: "Total calls to network processor execute work fn", + buckets: [0, 1, 5, 128], + }), + canNotAcceptWork: register.gauge({ + name: "lodestar_network_processor_can_not_accept_work_total", + help: "Total times network processor can not accept work on executeWork", + }), + }, + discv5: { decodeEnrAttemptCount: register.counter({ name: "lodestar_discv5_decode_enr_attempt_count", @@ -542,6 +558,10 @@ export function createLodestarMetrics( name: "lodestar_bls_thread_pool_queue_length", help: "Count of total block processor queue length", }), + workersBusy: register.gauge({ + name: "lodestar_bls_thread_pool_workers_busy", + help: "Count of current busy workers", + }), totalJobsGroupsStarted: register.gauge({ name: "lodestar_bls_thread_pool_job_groups_started_total", help: "Count of total jobs groups started in bls thread pool, job groups include +1 jobs", @@ -1129,6 +1149,29 @@ export function createLodestarMetrics( help: "Total times SeenContributionAndProof.isKnown returning true", }), }, + attestationData: { + totalSlot: register.gauge({ + name: "lodestar_seen_cache_attestation_data_slot_total", + help: "Total number of slots of attestation data in SeenAttestationData", + }), + countPerSlot: register.gauge({ + name: "lodestar_seen_cache_attestation_data_per_slot_total", + help: "Total number of attestation data per slot in SeenAttestationData", + }), + hit: register.gauge({ + name: "lodestar_seen_cache_attestation_data_hit_total", + help: "Total number of attestation data hit in SeenAttestationData", + }), + miss: register.gauge({ + name: "lodestar_seen_cache_attestation_data_miss_total", + help: "Total number of attestation data miss in SeenAttestationData", + }), + reject: register.gauge<"reason">({ + name: "lodestar_seen_cache_attestation_data_reject_total", + help: "Total number of attestation data rejected in SeenAttestationData", + labelNames: ["reason"], + }), + }, }, regenFnCallTotal: register.gauge<"entrypoint" | "caller">({ @@ -1175,7 +1218,7 @@ export function createLodestarMetrics( }, // reprocess attestations - reprocessAttestations: { + reprocessApiAttestations: { total: register.gauge({ name: "lodestar_reprocess_attestations_total", help: "Total number of attestations waiting to reprocess", @@ -1184,7 +1227,7 @@ export function createLodestarMetrics( name: "lodestar_reprocess_attestations_resolve_total", help: "Total number of attestations are reprocessed", }), - waitTimeBeforeResolve: register.gauge({ + waitSecBeforeResolve: register.gauge({ name: "lodestar_reprocess_attestations_wait_time_resolve_seconds", help: "Time to wait for unknown block in seconds", }), @@ -1193,12 +1236,41 @@ export function createLodestarMetrics( help: "Total number of attestations are rejected to reprocess", labelNames: ["reason"], }), - waitTimeBeforeReject: register.gauge<"reason">({ + waitSecBeforeReject: register.gauge<"reason">({ name: "lodestar_reprocess_attestations_wait_time_reject_seconds", help: "Time to wait for unknown block before being rejected", }), }, + // reprocess gossip attestations + reprocessGossipAttestations: { + total: register.gauge({ + name: "lodestar_reprocess_gossip_attestations_total", + help: "Total number of gossip attestations waiting to reprocess", + }), + countPerSlot: register.gauge({ + name: "lodestar_reprocess_gossip_attestations_per_slot_total", + help: "Total number of gossip attestations waiting to reprocess pet slot", + }), + resolve: register.gauge({ + name: "lodestar_reprocess_gossip_attestations_resolve_total", + help: "Total number of gossip attestations are reprocessed", + }), + waitSecBeforeResolve: register.gauge({ + name: "lodestar_reprocess_gossip_attestations_wait_time_resolve_seconds", + help: "Time to wait for unknown block in seconds", + }), + reject: register.gauge<"reason">({ + name: "lodestar_reprocess_gossip_attestations_reject_total", + help: "Total number of attestations are rejected to reprocess", + labelNames: ["reason"], + }), + waitSecBeforeReject: register.gauge<"reason">({ + name: "lodestar_reprocess_gossip_attestations_wait_time_reject_seconds", + help: "Time to wait for unknown block before being rejected", + }), + }, + lightclientServer: { onSyncAggregate: register.gauge<"event">({ name: "lodestar_lightclient_server_on_sync_aggregate_event_total", diff --git a/packages/beacon-node/src/network/events.ts b/packages/beacon-node/src/network/events.ts index 2973ca60a4e8..2111afaa4c04 100644 --- a/packages/beacon-node/src/network/events.ts +++ b/packages/beacon-node/src/network/events.ts @@ -1,9 +1,11 @@ import {EventEmitter} from "events"; import {PeerId} from "@libp2p/interface-peer-id"; import StrictEventEmitter from "strict-event-emitter-types"; +import {TopicValidatorResult} from "@libp2p/interface-pubsub"; import {phase0} from "@lodestar/types"; import {BlockInput} from "../chain/blocks/types.js"; import {RequestTypedContainer} from "./reqresp/ReqRespBeaconNode.js"; +import {PendingGossipsubMessage} from "./processor/types.js"; export enum NetworkEvent { /** A relevant peer has connected or has been re-STATUS'd */ @@ -14,6 +16,10 @@ export enum NetworkEvent { gossipHeartbeat = "gossipsub.heartbeat", reqRespRequest = "req-resp.request", unknownBlockParent = "unknownBlockParent", + + // Network processor events + pendingGossipsubMessage = "gossip.pendingGossipsubMessage", + gossipMessageValidationResult = "gossip.messageValidationResult", } export type NetworkEvents = { @@ -21,6 +27,12 @@ export type NetworkEvents = { [NetworkEvent.peerDisconnected]: (peer: PeerId) => void; [NetworkEvent.reqRespRequest]: (request: RequestTypedContainer, peer: PeerId) => void; [NetworkEvent.unknownBlockParent]: (blockInput: BlockInput, peerIdStr: string) => void; + [NetworkEvent.pendingGossipsubMessage]: (data: PendingGossipsubMessage) => void; + [NetworkEvent.gossipMessageValidationResult]: ( + msgId: string, + propagationSource: PeerId, + acceptance: TopicValidatorResult + ) => void; }; export type INetworkEventBus = StrictEventEmitter; diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts index 69f0e693dc7d..11adcde8d834 100644 --- a/packages/beacon-node/src/network/gossip/encoding.ts +++ b/packages/beacon-node/src/network/gossip/encoding.ts @@ -6,7 +6,7 @@ import {intToBytes, toHex} from "@lodestar/utils"; import {ForkName} from "@lodestar/params"; import {RPC} from "@chainsafe/libp2p-gossipsub/message"; import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js"; -import {GossipTopicCache} from "./topic.js"; +import {getGossipSSZType, GossipTopicCache} from "./topic.js"; // Load WASM const xxhash = await xxhashFactory(); @@ -62,7 +62,7 @@ export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8 } export class DataTransformSnappy { - constructor(private readonly maxSizePerMessage: number) {} + constructor(private readonly gossipTopicCache: GossipTopicCache, private readonly maxSizePerMessage: number) {} /** * Takes the data published by peers on a topic and transforms the data. @@ -71,9 +71,24 @@ export class DataTransformSnappy { * - `outboundTransform()`: compress snappy payload */ inboundTransform(topicStr: string, data: Uint8Array): Uint8Array { - // No need to parse topic, everything is snappy compressed - return uncompress(data, this.maxSizePerMessage); + const uncompressedData = uncompress(data, this.maxSizePerMessage); + + // check uncompressed data length before we extract beacon block root, slot or + // attestation data at later steps + const uncompressedDataLength = uncompressedData.length; + const topic = this.gossipTopicCache.getTopic(topicStr); + const sszType = getGossipSSZType(topic); + + if (uncompressedDataLength < sszType.minSize) { + throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} < ${sszType.minSize}`); + } + if (uncompressedDataLength > sszType.maxSize) { + throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${sszType.maxSize}`); + } + + return uncompressedData; } + /** * Takes the data to be published (a topic and associated data) transforms the data. The * transformed data will then be used to create a `RawGossipsubMessage` to be sent to peers. diff --git a/packages/beacon-node/src/network/gossip/gossipsub.ts b/packages/beacon-node/src/network/gossip/gossipsub.ts index f08975ab716d..1dc5ffdfd7a0 100644 --- a/packages/beacon-node/src/network/gossip/gossipsub.ts +++ b/packages/beacon-node/src/network/gossip/gossipsub.ts @@ -1,3 +1,5 @@ +import {PeerId} from "@libp2p/interface-peer-id"; +import {TopicValidatorResult} from "@libp2p/interface-pubsub"; import {GossipSub, GossipsubEvents} from "@chainsafe/libp2p-gossipsub"; import {PublishOpts, SignaturePolicy, TopicStr} from "@chainsafe/libp2p-gossipsub/types"; import {PeerScore, PeerScoreParams} from "@chainsafe/libp2p-gossipsub/score"; @@ -15,19 +17,10 @@ import {PeersData} from "../peers/peersData.js"; import {ClientKind} from "../peers/client.js"; import {GOSSIP_MAX_SIZE, GOSSIP_MAX_SIZE_BELLATRIX} from "../../constants/network.js"; import {Libp2p} from "../interface.js"; -import { - GossipJobQueues, - GossipTopic, - GossipTopicMap, - GossipType, - GossipTypeMap, - ValidatorFnsByType, - GossipHandlers, - GossipBeaconNode, -} from "./interface.js"; +import {NetworkEvent, NetworkEventBus} from "../events.js"; +import {GossipBeaconNode, GossipTopic, GossipTopicMap, GossipType, GossipTypeMap} from "./interface.js"; import {getGossipSSZType, GossipTopicCache, stringifyGossipTopic, getCoreTopicsAtFork} from "./topic.js"; import {DataTransformSnappy, fastMsgIdFn, msgIdFn, msgIdToStrFn} from "./encoding.js"; -import {createValidatorFnsByType} from "./validation/index.js"; import { computeGossipPeerScoreParams, @@ -48,10 +41,9 @@ export type Eth2GossipsubModules = { libp2p: Libp2p; logger: Logger; metrics: Metrics | null; - signal: AbortSignal; eth2Context: Eth2Context; - gossipHandlers: GossipHandlers; peersData: PeersData; + events: NetworkEventBus; }; export type Eth2GossipsubOpts = { @@ -60,6 +52,7 @@ export type Eth2GossipsubOpts = { gossipsubDLow?: number; gossipsubDHigh?: number; gossipsubAwaitHandler?: boolean; + skipParamsLog?: boolean; }; /** @@ -76,23 +69,21 @@ export type Eth2GossipsubOpts = { * See https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub */ export class Eth2Gossipsub extends GossipSub implements GossipBeaconNode { - readonly jobQueues: GossipJobQueues; readonly scoreParams: Partial; private readonly config: BeaconConfig; private readonly logger: Logger; private readonly peersData: PeersData; + private readonly events: NetworkEventBus; // Internal caches private readonly gossipTopicCache: GossipTopicCache; - private readonly validatorFnsByType: ValidatorFnsByType; - constructor(opts: Eth2GossipsubOpts, modules: Eth2GossipsubModules) { const {allowPublishToZeroPeers, gossipsubD, gossipsubDLow, gossipsubDHigh} = opts; const gossipTopicCache = new GossipTopicCache(modules.config); const scoreParams = computeGossipPeerScoreParams(modules); - const {config, logger, metrics, signal, gossipHandlers, peersData} = modules; + const {config, logger, metrics, peersData, events} = modules; // Gossipsub parameters defined here: // https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub @@ -125,6 +116,7 @@ export class Eth2Gossipsub extends GossipSub implements GossipBeaconNode { // // TODO: figure out a way to dynamically transition to the size dataTransform: new DataTransformSnappy( + gossipTopicCache, isFinite(config.BELLATRIX_FORK_EPOCH) ? GOSSIP_MAX_SIZE_BELLATRIX : GOSSIP_MAX_SIZE ), metricsRegister: modules.metrics ? (modules.metrics.register as unknown as MetricsRegister) : null, @@ -137,29 +129,21 @@ export class Eth2Gossipsub extends GossipSub implements GossipBeaconNode { this.config = config; this.logger = logger; this.peersData = peersData; + this.events = events; this.gossipTopicCache = gossipTopicCache; - // Note: We use the validator functions as handlers. No handler will be registered to gossipsub. - // libp2p-js layer will emit the message to an EventEmitter that won't be listened by anyone. - // TODO: Force to ensure there's a validatorFunction attached to every received topic. - const {validatorFnsByType, jobQueues} = createValidatorFnsByType(gossipHandlers, { - config, - logger, - metrics, - signal, - }); - this.validatorFnsByType = validatorFnsByType; - this.jobQueues = jobQueues; - if (metrics) { metrics.gossipMesh.peersByType.addCollect(() => this.onScrapeLodestarMetrics(metrics)); } this.addEventListener("gossipsub:message", this.onGossipsubMessage.bind(this)); + this.events.on(NetworkEvent.gossipMessageValidationResult, this.onValidationResult.bind(this)); // Having access to this data is CRUCIAL for debugging. While this is a massive log, it must not be deleted. // Scoring issues require this dump + current peer score stats to re-calculate scores. - this.logger.debug("Gossipsub score params", {params: JSON.stringify(scoreParams)}); + if (!opts.skipParamsLog) { + this.logger.debug("Gossipsub score params", {params: JSON.stringify(scoreParams)}); + } } /** @@ -422,14 +406,23 @@ export class Eth2Gossipsub extends GossipSub implements GossipBeaconNode { // Get seenTimestamp before adding the message to the queue or add async delays const seenTimestampSec = Date.now() / 1000; - // Puts object in queue, validates, then processes - this.validatorFnsByType[topic.type](topic, msg, propagationSource.toString(), seenTimestampSec) - .then((acceptance) => { - this.reportMessageValidationResult(msgId, propagationSource, acceptance); - }) - .catch((e) => { - this.logger.error("Error onGossipsubMessage", {}, e); + // Emit message to network processor, use setTimeout to yield to the macro queue + // This is mostly due to too many attestation messages, and a gossipsub RPC may + // contain multiple of them. This helps avoid the I/O lag issue. + setTimeout(() => { + this.events.emit(NetworkEvent.pendingGossipsubMessage, { + topic, + msg, + msgId, + propagationSource, + seenTimestampSec, + startProcessUnixSec: null, }); + }, 0); + } + + private onValidationResult(msgId: string, propagationSource: PeerId, acceptance: TopicValidatorResult): void { + this.reportMessageValidationResult(msgId, propagationSource, acceptance); } } diff --git a/packages/beacon-node/src/network/gossip/index.ts b/packages/beacon-node/src/network/gossip/index.ts index 07d3d1310a3c..e76bc9279444 100644 --- a/packages/beacon-node/src/network/gossip/index.ts +++ b/packages/beacon-node/src/network/gossip/index.ts @@ -1,4 +1,4 @@ export {Eth2Gossipsub} from "./gossipsub.js"; -export {getGossipHandlers} from "./handlers/index.js"; +export {getGossipHandlers} from "../processor/gossipHandlers.js"; export {getCoreTopicsAtFork} from "./topic.js"; export * from "./interface.js"; diff --git a/packages/beacon-node/src/network/gossip/interface.ts b/packages/beacon-node/src/network/gossip/interface.ts index 4fbc550f457e..d0dbd998674e 100644 --- a/packages/beacon-node/src/network/gossip/interface.ts +++ b/packages/beacon-node/src/network/gossip/interface.ts @@ -165,14 +165,16 @@ export type GossipHandlerFn = ( object: GossipTypeMap[GossipType], topic: GossipTopicMap[GossipType], peerIdStr: string, - seenTimestampSec: number + seenTimestampSec: number, + gossipSerializedData: Uint8Array ) => Promise; export type GossipHandlers = { [K in GossipType]: ( object: GossipTypeMap[K], topic: GossipTopicMap[K], peerIdStr: string, - seenTimestampSec: number + seenTimestampSec: number, + gossipSerializedData: Uint8Array ) => Promise; }; diff --git a/packages/beacon-node/src/network/gossip/validation/onAccept.ts b/packages/beacon-node/src/network/gossip/validation/onAccept.ts deleted file mode 100644 index 810e7ccfd64b..000000000000 --- a/packages/beacon-node/src/network/gossip/validation/onAccept.ts +++ /dev/null @@ -1,15 +0,0 @@ -import {ChainForkConfig} from "@lodestar/config"; -import {GossipType, GossipTypeMap, GossipTopicTypeMap} from "../interface.js"; - -export type GetGossipAcceptMetadataFn = ( - config: ChainForkConfig, - object: GossipTypeMap[GossipType], - topic: GossipTopicTypeMap[GossipType] -) => Record; -export type GetGossipAcceptMetadataFns = { - [K in GossipType]: ( - config: ChainForkConfig, - object: GossipTypeMap[K], - topic: GossipTopicTypeMap[K] - ) => Record; -}; diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index 5ba8998ba748..3a3da939ff77 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -10,10 +10,11 @@ import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; import {routes} from "@lodestar/api"; import {BlockInput} from "../chain/blocks/types.js"; import {INetworkEventBus} from "./events.js"; -import {GossipBeaconNode} from "./gossip/index.js"; +import {GossipBeaconNode, GossipType} from "./gossip/index.js"; import {PeerAction, PeerScoreStats} from "./peers/index.js"; import {IReqRespBeaconNode} from "./reqresp/ReqRespBeaconNode.js"; import {AttnetsService, CommitteeSubscription} from "./subnets/index.js"; +import {PendingGossipsubMessage} from "./processor/types.js"; export type PeerSearchOptions = { supportsProtocols?: string[]; @@ -62,7 +63,7 @@ export interface INetwork { dumpPeer(peerIdStr: string): Promise; dumpPeerScoreStats(): Promise; dumpGossipPeerScoreStats(): Promise; - dumpGossipQueueItems(gossipType: string): Promise; + dumpGossipQueue(gossipType: GossipType): Promise; dumpDiscv5KadValues(): Promise; } diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index 4b69e324d819..0fac53cbf1c2 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -20,7 +20,6 @@ import {ReqRespBeaconNode, ReqRespHandlers, beaconBlocksMaybeBlobsByRange} from import {beaconBlocksMaybeBlobsByRoot} from "./reqresp/beaconBlocksMaybeBlobsByRoot.js"; import { Eth2Gossipsub, - getGossipHandlers, GossipHandlers, GossipTopicTypeMap, GossipType, @@ -37,6 +36,8 @@ import {PeersData} from "./peers/peersData.js"; import {getConnectionsMap, isPublishToZeroPeersError} from "./util.js"; import {Discv5Worker} from "./discv5/index.js"; import {createNodeJsLibp2p} from "./nodejs/util.js"; +import {NetworkProcessor} from "./processor/index.js"; +import {PendingGossipsubMessage} from "./processor/types.js"; // How many changes to batch cleanup const CACHED_BLS_BATCH_CLEANUP_LIMIT = 10; @@ -50,6 +51,7 @@ type NetworkModules = { signal: AbortSignal; peersData: PeersData; networkEventBus: NetworkEventBus; + networkProcessor: NetworkProcessor; metadata: MetadataController; peerRpcScores: PeerRpcScoreStore; reqResp: ReqRespBeaconNode; @@ -84,6 +86,7 @@ export class Network implements INetwork { private readonly opts: NetworkOptions; private readonly peersData: PeersData; + private readonly networkProcessor: NetworkProcessor; private readonly peerManager: PeerManager; private readonly libp2p: Libp2p; private readonly logger: Logger; @@ -106,6 +109,7 @@ export class Network implements INetwork { signal, peersData, networkEventBus, + networkProcessor, metadata, peerRpcScores, reqResp, @@ -123,7 +127,7 @@ export class Network implements INetwork { this.signal = signal; this.peersData = peersData; this.events = networkEventBus; - this.metadata = metadata; + (this.networkProcessor = networkProcessor), (this.metadata = metadata); this.peerRpcScores = peerRpcScores; this.reqResp = reqResp; this.gossip = gossip; @@ -146,8 +150,8 @@ export class Network implements INetwork { peerStoreDir, chain, reqRespHandlers, - gossipHandlers, signal, + gossipHandlers, }: NetworkInitModules): Promise { const clock = chain.clock; const peersData = new PeersData(); @@ -196,16 +200,13 @@ export class Network implements INetwork { libp2p, logger, metrics, - signal, - gossipHandlers: - gossipHandlers ?? - getGossipHandlers({chain, config, logger, attnetsService, peerRpcScores, networkEventBus, metrics}, opts), eth2Context: { activeValidatorCount: chain.getHeadState().epochCtx.currentShuffling.activeIndices.length, currentSlot: clock.currentSlot, currentEpoch: clock.currentEpoch, }, peersData, + events: networkEventBus, }); const syncnetsService = new SyncnetsService(config, chain, gossip, metadata, logger, metrics, opts); @@ -228,6 +229,11 @@ export class Network implements INetwork { opts ); + const networkProcessor = new NetworkProcessor( + {attnetsService, chain, config, logger, metrics, peerRpcScores, events: networkEventBus, gossipHandlers}, + opts + ); + await libp2p.start(); // Network spec decides version changes based on clock fork, not head fork @@ -260,6 +266,7 @@ export class Network implements INetwork { signal, peersData, networkEventBus, + networkProcessor, metadata, peerRpcScores, reqResp, @@ -411,9 +418,7 @@ export class Network implements INetwork { } // Drop all the gossip validation queues - for (const jobQueue of Object.values(this.gossip.jobQueues)) { - jobQueue.dropAllJobs(); - } + this.networkProcessor.dropAllJobs(); } isSubscribedToGossipCoreTopics(): boolean { @@ -445,24 +450,6 @@ export class Network implements INetwork { })); } - async dumpGossipQueueItems(gossipType: string): Promise { - const jobQueue = this.gossip.jobQueues[gossipType as GossipType]; - if (jobQueue === undefined) { - throw Error(`Unknown gossipType ${gossipType}, known values: ${Object.keys(jobQueue).join(", ")}`); - } - - return jobQueue.getItems().map((item) => { - const [topic, message, propagationSource, seenTimestampSec] = item.args; - return { - topic: topic, - propagationSource, - data: message.data, - addedTimeMs: item.addedTimeMs, - seenTimestampSec, - }; - }); - } - async dumpPeerScoreStats(): Promise { return this.peerRpcScores.dumpPeerScoreStats(); } @@ -475,6 +462,10 @@ export class Network implements INetwork { return (await this.discv5?.kadValues())?.map((enr) => enr.encodeTxt()) ?? []; } + async dumpGossipQueue(gossipType: GossipType): Promise { + return this.networkProcessor.dumpGossipQueue(gossipType); + } + /** * Handle subscriptions through fork transitions, @see FORK_EPOCH_LOOKAHEAD */ diff --git a/packages/beacon-node/src/network/options.ts b/packages/beacon-node/src/network/options.ts index e004d671597c..696bb65f3876 100644 --- a/packages/beacon-node/src/network/options.ts +++ b/packages/beacon-node/src/network/options.ts @@ -1,15 +1,16 @@ import {generateKeypair, IDiscv5DiscoveryInputOptions, KeypairType, SignableENR} from "@chainsafe/discv5"; import {Eth2GossipsubOpts} from "./gossip/gossipsub.js"; -import {defaultGossipHandlerOpts, GossipHandlerOpts} from "./gossip/handlers/index.js"; +import {defaultGossipHandlerOpts} from "./processor/gossipHandlers.js"; import {PeerManagerOpts} from "./peers/index.js"; import {ReqRespBeaconNodeOpts} from "./reqresp/ReqRespBeaconNode.js"; +import {NetworkProcessorOpts} from "./processor/index.js"; // Since Network is eventually intended to be run in a separate thread, ensure that all options are cloneable using structuredClone export interface NetworkOptions extends PeerManagerOpts, // remove all Functions Omit, - GossipHandlerOpts, + NetworkProcessorOpts, Eth2GossipsubOpts { localMultiaddrs: string[]; bootMultiaddrs?: string[]; diff --git a/packages/beacon-node/src/network/processor/extractSlotRootFns.ts b/packages/beacon-node/src/network/processor/extractSlotRootFns.ts new file mode 100644 index 000000000000..5a8a0962a292 --- /dev/null +++ b/packages/beacon-node/src/network/processor/extractSlotRootFns.ts @@ -0,0 +1,36 @@ +import {SlotRootHex} from "@lodestar/types"; +import { + getBlockRootFromAttestationSerialized, + getBlockRootFromSignedAggregateAndProofSerialized, + getSlotFromAttestationSerialized, + getSlotFromSignedAggregateAndProofSerialized, +} from "../../util/sszBytes.js"; +import {GossipType} from "../gossip/index.js"; +import {ExtractSlotRootFns} from "./types.js"; + +/** + * Extract the slot and block root of a gossip message form serialized data. + * Only applicable for beacon_attestation and beacon_aggregate_and_proof topics. + */ +export function createExtractBlockSlotRootFns(): ExtractSlotRootFns { + return { + [GossipType.beacon_attestation]: (data: Uint8Array): SlotRootHex | null => { + const slot = getSlotFromAttestationSerialized(data); + const root = getBlockRootFromAttestationSerialized(data); + + if (slot === null || root === null) { + return null; + } + return {slot, root}; + }, + [GossipType.beacon_aggregate_and_proof]: (data: Uint8Array): SlotRootHex | null => { + const slot = getSlotFromSignedAggregateAndProofSerialized(data); + const root = getBlockRootFromSignedAggregateAndProofSerialized(data); + + if (slot === null || root === null) { + return null; + } + return {slot, root}; + }, + }; +} diff --git a/packages/beacon-node/src/network/gossip/handlers/index.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts similarity index 88% rename from packages/beacon-node/src/network/gossip/handlers/index.ts rename to packages/beacon-node/src/network/processor/gossipHandlers.ts index 8f04057b5c9f..8cf01e62d81f 100644 --- a/packages/beacon-node/src/network/gossip/handlers/index.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -2,11 +2,11 @@ import {peerIdFromString} from "@libp2p/peer-id"; import {toHexString} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; import {Logger, prettyBytes} from "@lodestar/utils"; -import {phase0, Root, Slot, ssz} from "@lodestar/types"; +import {Root, Slot, ssz} from "@lodestar/types"; import {ForkName, ForkSeq} from "@lodestar/params"; -import {Metrics} from "../../../metrics/index.js"; -import {OpSource} from "../../../metrics/validatorMonitor.js"; -import {IBeaconChain} from "../../../chain/index.js"; +import {Metrics} from "../../metrics/index.js"; +import {OpSource} from "../../metrics/validatorMonitor.js"; +import {IBeaconChain} from "../../chain/index.js"; import { AttestationError, AttestationErrorCode, @@ -16,8 +16,8 @@ import { GossipAction, GossipActionError, SyncCommitteeError, -} from "../../../chain/errors/index.js"; -import {GossipHandlers, GossipType} from "../interface.js"; +} from "../../chain/errors/index.js"; +import {GossipHandlers, GossipType} from "../gossip/interface.js"; import { validateGossipAggregateAndProof, validateGossipAttestation, @@ -28,14 +28,16 @@ import { validateSyncCommitteeGossipContributionAndProof, validateGossipVoluntaryExit, validateBlsToExecutionChange, -} from "../../../chain/validation/index.js"; -import {NetworkEvent, NetworkEventBus} from "../../events.js"; -import {PeerAction, PeerRpcScoreStore} from "../../peers/index.js"; -import {validateLightClientFinalityUpdate} from "../../../chain/validation/lightClientFinalityUpdate.js"; -import {validateLightClientOptimisticUpdate} from "../../../chain/validation/lightClientOptimisticUpdate.js"; -import {validateGossipBlobsSidecar} from "../../../chain/validation/blobsSidecar.js"; -import {BlockInput, getBlockInput} from "../../../chain/blocks/types.js"; -import {AttnetsService} from "../../subnets/attnetsService.js"; + AttestationValidationResult, + AggregateAndProofValidationResult, +} from "../../chain/validation/index.js"; +import {NetworkEvent, NetworkEventBus} from "../events.js"; +import {PeerAction, PeerRpcScoreStore} from "../peers/index.js"; +import {validateLightClientFinalityUpdate} from "../../chain/validation/lightClientFinalityUpdate.js"; +import {validateLightClientOptimisticUpdate} from "../../chain/validation/lightClientOptimisticUpdate.js"; +import {validateGossipBlobsSidecar} from "../../chain/validation/blobsSidecar.js"; +import {BlockInput, getBlockInput} from "../../chain/blocks/types.js"; +import {AttnetsService} from "../subnets/attnetsService.js"; /** * Gossip handler options as part of network options @@ -52,13 +54,13 @@ export const defaultGossipHandlerOpts = { dontSendGossipAttestationsToForkchoice: false, }; -type ValidatorFnsModules = { +export type ValidatorFnsModules = { attnetsService: AttnetsService; chain: IBeaconChain; config: BeaconConfig; logger: Logger; metrics: Metrics | null; - networkEventBus: NetworkEventBus; + events: NetworkEventBus; peerRpcScores: PeerRpcScoreStore; }; @@ -79,7 +81,7 @@ const MAX_UNKNOWN_BLOCK_ROOT_RETRIES = 1; * - Ethereum Consensus gossipsub protocol strictly defined a single topic for message */ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipHandlerOpts): GossipHandlers { - const {attnetsService, chain, config, metrics, networkEventBus, peerRpcScores, logger} = modules; + const {attnetsService, chain, config, metrics, events, peerRpcScores, logger} = modules; async function validateBeaconBlock( blockInput: BlockInput, @@ -109,7 +111,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH if (e instanceof BlockGossipError) { if (e instanceof BlockGossipError && e.type.code === BlockErrorCode.PARENT_UNKNOWN) { logger.debug("Gossip block has error", {slot, root: blockHex, code: e.type.code}); - networkEventBus.emit(NetworkEvent.unknownBlockParent, blockInput, peerIdStr); + events.emit(NetworkEvent.unknownBlockParent, blockInput, peerIdStr); } } @@ -191,15 +193,23 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec); }, - [GossipType.beacon_aggregate_and_proof]: async (signedAggregateAndProof, _topic, _peer, seenTimestampSec) => { - let validationResult: {indexedAttestation: phase0.IndexedAttestation; committeeIndices: number[]}; + [GossipType.beacon_aggregate_and_proof]: async ( + signedAggregateAndProof, + _topic, + _peer, + seenTimestampSec, + gossipSerializedData + ) => { + let validationResult: AggregateAndProofValidationResult; + try { // If an attestation refers to a block root that's not known, it will wait for 1 slot max // See https://github.com/ChainSafe/lodestar/pull/3564 for reasoning and results // Waiting here requires minimal code and automatically affects attestation, and aggregate validation // both from gossip and the API. I also prevents having to catch and re-throw in multiple places. // eslint-disable-next-line @typescript-eslint/explicit-function-return-type - const validateFn = () => validateGossipAggregateAndProof(chain, signedAggregateAndProof); + const validateFn = () => + validateGossipAggregateAndProof(chain, signedAggregateAndProof, false, gossipSerializedData); const {slot, beaconBlockRoot} = signedAggregateAndProof.message.aggregate.data; validationResult = await validateGossipFnRetryUnknownRoot(validateFn, chain, slot, beaconBlockRoot); } catch (e) { @@ -210,19 +220,20 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH } // Handler - const {indexedAttestation, committeeIndices} = validationResult; + const {indexedAttestation, committeeIndices, attDataRootHex} = validationResult; metrics?.registerGossipAggregatedAttestation(seenTimestampSec, signedAggregateAndProof, indexedAttestation); const aggregatedAttestation = signedAggregateAndProof.message.aggregate; chain.aggregatedAttestationPool.add( aggregatedAttestation, + attDataRootHex, indexedAttestation.attestingIndices.length, committeeIndices ); if (!options.dontSendGossipAttestationsToForkchoice) { try { - chain.forkChoice.onAttestation(indexedAttestation); + chain.forkChoice.onAttestation(indexedAttestation, attDataRootHex); } catch (e) { logger.debug( "Error adding gossip aggregated attestation to forkchoice", @@ -233,11 +244,11 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH } }, - [GossipType.beacon_attestation]: async (attestation, {subnet}, _peer, seenTimestampSec) => { - let validationResult: {indexedAttestation: phase0.IndexedAttestation; subnet: number}; + [GossipType.beacon_attestation]: async (attestation, {subnet}, _peer, seenTimestampSec, gossipSerializedData) => { + let validationResult: AttestationValidationResult; try { // eslint-disable-next-line @typescript-eslint/explicit-function-return-type - const validateFn = () => validateGossipAttestation(chain, attestation, subnet); + const validateFn = () => validateGossipAttestation(chain, attestation, subnet, gossipSerializedData); const {slot, beaconBlockRoot} = attestation.data; // If an attestation refers to a block root that's not known, it will wait for 1 slot max // See https://github.com/ChainSafe/lodestar/pull/3564 for reasoning and results @@ -252,14 +263,14 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH } // Handler - const {indexedAttestation} = validationResult; + const {indexedAttestation, attDataRootHex} = validationResult; metrics?.registerGossipUnaggregatedAttestation(seenTimestampSec, indexedAttestation); try { // Node may be subscribe to extra subnets (long-lived random subnets). For those, validate the messages // but don't add to attestation pool, to save CPU and RAM if (attnetsService.shouldProcess(subnet, attestation.data.slot)) { - const insertOutcome = chain.attestationPool.add(attestation); + const insertOutcome = chain.attestationPool.add(attestation, attDataRootHex); metrics?.opPool.attestationPoolInsertOutcome.inc({insertOutcome}); } } catch (e) { @@ -268,7 +279,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH if (!options.dontSendGossipAttestationsToForkchoice) { try { - chain.forkChoice.onAttestation(indexedAttestation); + chain.forkChoice.onAttestation(indexedAttestation, attDataRootHex); } catch (e) { logger.debug("Error adding gossip unaggregated attestation to forkchoice", {subnet}, e as Error); } diff --git a/packages/beacon-node/src/network/gossip/validation/queue.ts b/packages/beacon-node/src/network/processor/gossipQueues.ts similarity index 61% rename from packages/beacon-node/src/network/gossip/validation/queue.ts rename to packages/beacon-node/src/network/processor/gossipQueues.ts index fbc43dadfca0..bd4f85cc1761 100644 --- a/packages/beacon-node/src/network/gossip/validation/queue.ts +++ b/packages/beacon-node/src/network/processor/gossipQueues.ts @@ -1,33 +1,89 @@ import {mapValues} from "@lodestar/utils"; -import {Metrics} from "../../../metrics/index.js"; -import {JobItemQueue, JobQueueOpts, QueueType} from "../../../util/queue/index.js"; -import {GossipJobQueues, GossipType, GossipValidatorFn, ResolvedType, ValidatorFnsByType} from "../interface.js"; +import {LinkedList} from "../../util/array.js"; +import {GossipType} from "../gossip/interface.js"; + +enum QueueType { + FIFO = "FIFO", + LIFO = "LIFO", +} /** * Numbers from https://github.com/sigp/lighthouse/blob/b34a79dc0b02e04441ba01fd0f304d1e203d877d/beacon_node/network/src/beacon_processor/mod.rs#L69 */ const gossipQueueOpts: { - [K in GossipType]: Pick; + [K in GossipType]: GossipQueueOpts; } = { // validation gossip block asap - [GossipType.beacon_block]: {maxLength: 1024, type: QueueType.FIFO, noYieldIfOneItem: true}, + [GossipType.beacon_block]: {maxLength: 1024, type: QueueType.FIFO}, // TODO DENEB: What's a good queue max given that now blocks are much bigger? - [GossipType.beacon_block_and_blobs_sidecar]: {maxLength: 32, type: QueueType.FIFO, noYieldIfOneItem: true}, + [GossipType.beacon_block_and_blobs_sidecar]: {maxLength: 32, type: QueueType.FIFO}, // lighthoue has aggregate_queue 4096 and unknown_block_aggregate_queue 1024, we use single queue - [GossipType.beacon_aggregate_and_proof]: {maxLength: 5120, type: QueueType.LIFO, maxConcurrency: 16}, + [GossipType.beacon_aggregate_and_proof]: {maxLength: 5120, type: QueueType.LIFO}, // lighthouse has attestation_queue 16384 and unknown_block_attestation_queue 8192, we use single queue - [GossipType.beacon_attestation]: {maxLength: 24576, type: QueueType.LIFO, maxConcurrency: 64}, + [GossipType.beacon_attestation]: {maxLength: 24576, type: QueueType.LIFO}, [GossipType.voluntary_exit]: {maxLength: 4096, type: QueueType.FIFO}, [GossipType.proposer_slashing]: {maxLength: 4096, type: QueueType.FIFO}, [GossipType.attester_slashing]: {maxLength: 4096, type: QueueType.FIFO}, - [GossipType.sync_committee_contribution_and_proof]: {maxLength: 4096, type: QueueType.LIFO, maxConcurrency: 16}, - [GossipType.sync_committee]: {maxLength: 4096, type: QueueType.LIFO, maxConcurrency: 64}, + [GossipType.sync_committee_contribution_and_proof]: {maxLength: 4096, type: QueueType.LIFO}, + [GossipType.sync_committee]: {maxLength: 4096, type: QueueType.LIFO}, [GossipType.light_client_finality_update]: {maxLength: 1024, type: QueueType.FIFO}, [GossipType.light_client_optimistic_update]: {maxLength: 1024, type: QueueType.FIFO}, // lighthouse has bls changes queue set to their max 16384 to handle large spike at capella [GossipType.bls_to_execution_change]: {maxLength: 16384, type: QueueType.FIFO}, }; +type GossipQueueOpts = { + type: QueueType; + maxLength: number; +}; + +export class GossipQueue { + private readonly list = new LinkedList(); + + constructor(private readonly opts: GossipQueueOpts) {} + + get length(): number { + return this.list.length; + } + + clear(): void { + this.list.clear(); + } + + add(item: T): T | null { + let droppedItem: T | null = null; + + if (this.list.length + 1 > this.opts.maxLength) { + // LIFO -> keep latest job, drop oldest, FIFO -> drop latest job + switch (this.opts.type) { + case QueueType.LIFO: + droppedItem = this.list.shift(); + break; + case QueueType.FIFO: + return item; + } + } + + this.list.push(item); + + return droppedItem; + } + + next(): T | null { + // LIFO -> pop() remove last item, FIFO -> shift() remove first item + switch (this.opts.type) { + case QueueType.LIFO: + return this.list.pop(); + case QueueType.FIFO: + return this.list.shift(); + } + } + + getAll(): T[] { + return this.list.toArray(); + } +} + /** * Wraps a GossipValidatorFn with a queue, to limit the processing of gossip objects by type. * @@ -44,25 +100,8 @@ const gossipQueueOpts: { * By topic is too specific, so by type groups all similar objects in the same queue. All in the same won't allow * to customize different queue behaviours per object type (see `gossipQueueOpts`). */ -export function createValidationQueues( - gossipValidatorFns: ValidatorFnsByType, - signal: AbortSignal, - metrics: Metrics | null -): GossipJobQueues { - return mapValues(gossipQueueOpts, (opts, type) => { - const gossipValidatorFn = gossipValidatorFns[type]; - return new JobItemQueue, ResolvedType>( - gossipValidatorFn, - {signal, ...opts}, - metrics - ? { - length: metrics.gossipValidationQueueLength.child({topic: type}), - droppedJobs: metrics.gossipValidationQueueDroppedJobs.child({topic: type}), - jobTime: metrics.gossipValidationQueueJobTime.child({topic: type}), - jobWaitTime: metrics.gossipValidationQueueJobWaitTime.child({topic: type}), - concurrency: metrics.gossipValidationQueueConcurrency.child({topic: type}), - } - : undefined - ); +export function createGossipQueues(): {[K in GossipType]: GossipQueue} { + return mapValues(gossipQueueOpts, (opts) => { + return new GossipQueue(opts); }); } diff --git a/packages/beacon-node/src/network/gossip/validation/index.ts b/packages/beacon-node/src/network/processor/gossipValidatorFn.ts similarity index 60% rename from packages/beacon-node/src/network/gossip/validation/index.ts rename to packages/beacon-node/src/network/processor/gossipValidatorFn.ts index 3f1a5f20742c..31b5ed175a3c 100644 --- a/packages/beacon-node/src/network/gossip/validation/index.ts +++ b/packages/beacon-node/src/network/processor/gossipValidatorFn.ts @@ -1,49 +1,17 @@ import {TopicValidatorResult} from "@libp2p/interface-pubsub"; import {ChainForkConfig} from "@lodestar/config"; -import {Logger, mapValues} from "@lodestar/utils"; -import {Metrics} from "../../../metrics/index.js"; -import {getGossipSSZType} from "../topic.js"; -import { - GossipJobQueues, - GossipType, - GossipValidatorFn, - ValidatorFnsByType, - GossipHandlers, - GossipHandlerFn, -} from "../interface.js"; -import {GossipActionError, GossipAction} from "../../../chain/errors/index.js"; -import {createValidationQueues} from "./queue.js"; +import {Logger} from "@lodestar/utils"; +import {Metrics} from "../../metrics/index.js"; +import {getGossipSSZType} from "../gossip/topic.js"; +import {GossipValidatorFn, GossipHandlers, GossipHandlerFn} from "../gossip/interface.js"; +import {GossipActionError, GossipAction} from "../../chain/errors/index.js"; -type ValidatorFnModules = { +export type ValidatorFnModules = { config: ChainForkConfig; logger: Logger; metrics: Metrics | null; }; -/** - * Returns GossipValidatorFn for each GossipType, given GossipHandlerFn indexed by type. - * - * @see getGossipHandlers for reasoning on why GossipHandlerFn are used for gossip validation. - */ -export function createValidatorFnsByType( - gossipHandlers: GossipHandlers, - modules: ValidatorFnModules & {signal: AbortSignal} -): {validatorFnsByType: ValidatorFnsByType; jobQueues: GossipJobQueues} { - const gossipValidatorFns = mapValues(gossipHandlers, (gossipHandler, type) => { - return getGossipValidatorFn(gossipHandler, type, modules); - }); - - const jobQueues = createValidationQueues(gossipValidatorFns, modules.signal, modules.metrics); - - const validatorFnsByType = mapValues(jobQueues, (jobQueue): GossipValidatorFn => { - return async function gossipValidatorFnWithQueue(topic, gossipMsg, propagationSource, seenTimestampSec) { - return jobQueue.push(topic, gossipMsg, propagationSource, seenTimestampSec); - }; - }); - - return {jobQueues, validatorFnsByType}; -} - /** * Returns a GossipSub validator function from a GossipHandlerFn. GossipHandlerFn may throw GossipActionError if one * or more validation conditions from the consensus-specs#p2p-interface are not satisfied. @@ -58,14 +26,12 @@ export function createValidatorFnsByType( * * @see getGossipHandlers for reasoning on why GossipHandlerFn are used for gossip validation. */ -function getGossipValidatorFn( - gossipHandler: GossipHandlers[K], - type: K, - modules: ValidatorFnModules -): GossipValidatorFn { +export function getGossipValidatorFn(gossipHandlers: GossipHandlers, modules: ValidatorFnModules): GossipValidatorFn { const {logger, metrics} = modules; return async function gossipValidatorFn(topic, msg, propagationSource, seenTimestampSec) { + const type = topic.type; + // Define in scope above try {} to be used in catch {} if object was parsed let gossipObject; try { @@ -78,7 +44,13 @@ function getGossipValidatorFn( return TopicValidatorResult.Reject; } - await (gossipHandler as GossipHandlerFn)(gossipObject, topic, propagationSource, seenTimestampSec); + await (gossipHandlers[type] as GossipHandlerFn)( + gossipObject, + topic, + propagationSource, + seenTimestampSec, + msg.data + ); metrics?.gossipValidationAccept.inc({topic: type}); diff --git a/packages/beacon-node/src/network/processor/index.ts b/packages/beacon-node/src/network/processor/index.ts new file mode 100644 index 000000000000..8079c5cc23b2 --- /dev/null +++ b/packages/beacon-node/src/network/processor/index.ts @@ -0,0 +1,297 @@ +import {Logger, MapDef, mapValues, sleep} from "@lodestar/utils"; +import {RootHex, Slot} from "@lodestar/types"; +import {routes} from "@lodestar/api"; +import {IBeaconChain} from "../../chain/interface.js"; +import {Metrics} from "../../metrics/metrics.js"; +import {NetworkEvent, NetworkEventBus} from "../events.js"; +import {GossipType} from "../gossip/interface.js"; +import {ChainEvent} from "../../chain/emitter.js"; +import {createGossipQueues} from "./gossipQueues.js"; +import {NetworkWorker, NetworkWorkerModules} from "./worker.js"; +import {PendingGossipsubMessage} from "./types.js"; +import {ValidatorFnsModules, GossipHandlerOpts} from "./gossipHandlers.js"; +import {createExtractBlockSlotRootFns} from "./extractSlotRootFns.js"; + +export type NetworkProcessorModules = NetworkWorkerModules & + ValidatorFnsModules & { + chain: IBeaconChain; + events: NetworkEventBus; + logger: Logger; + metrics: Metrics | null; + }; + +export type NetworkProcessorOpts = GossipHandlerOpts & { + maxGossipTopicConcurrency?: number; +}; + +type WorkOpts = { + bypassQueue?: boolean; +}; + +/** + * True if we want to process gossip object immediately, false if we check for bls and regen + * in order to process the gossip object. + */ +const executeGossipWorkOrderObj: Record = { + [GossipType.beacon_block]: {bypassQueue: true}, + [GossipType.beacon_block_and_blobs_sidecar]: {bypassQueue: true}, + [GossipType.beacon_aggregate_and_proof]: {}, + [GossipType.beacon_attestation]: {}, + [GossipType.voluntary_exit]: {}, + [GossipType.proposer_slashing]: {}, + [GossipType.attester_slashing]: {}, + [GossipType.sync_committee_contribution_and_proof]: {}, + [GossipType.sync_committee]: {}, + [GossipType.light_client_finality_update]: {}, + [GossipType.light_client_optimistic_update]: {}, + [GossipType.bls_to_execution_change]: {}, +}; +const executeGossipWorkOrder = Object.keys(executeGossipWorkOrderObj) as (keyof typeof executeGossipWorkOrderObj)[]; + +// TODO: Arbitrary constant, check metrics +const MAX_JOBS_SUBMITTED_PER_TICK = 128; + +// How many attestations (aggregate + unaggregate) we keep before new ones get dropped. +const MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS = 16_384; + +// We don't want to process too many attestations in a single tick +// As seen on mainnet, attestation concurrency metric ranges from 1000 to 2000 +// so make this constant a little bit conservative +const MAX_UNKNOWN_BLOCK_GOSSIP_OBJECTS_PER_TICK = 1024; + +// Same motivation to JobItemQueue, we don't want to block the event loop +const PROCESS_UNKNOWN_BLOCK_GOSSIP_OBJECTS_YIELD_EVERY_MS = 50; + +/** + * Reprocess reject reason for metrics + */ +enum ReprocessRejectReason { + /** + * There are too many attestations that have unknown block root. + */ + reached_limit = "reached_limit", + /** + * The awaiting attestation is pruned per clock slot. + */ + expired = "expired", +} + +/** + * Network processor handles the gossip queues and throtles processing to not overload the main thread + * - Decides when to process work and what to process + * + * What triggers execute work? + * + * - When work is submitted + * - When downstream workers become available + * + * ### PendingGossipsubMessage beacon_attestation example + * + * For attestations, processing the message includes the steps: + * 1. Pre shuffling sync validation + * 2. Retrieve shuffling: async + goes into the regen queue and can be expensive + * 3. Pre sig validation sync validation + * 4. Validate BLS signature: async + goes into workers through another manager + * + * The gossip queues should receive "backpressue" from the regen and BLS workers queues. + * Such that enough work is processed to fill either one of the queue. + */ +export class NetworkProcessor { + private readonly worker: NetworkWorker; + private readonly chain: IBeaconChain; + private readonly events: NetworkEventBus; + private readonly logger: Logger; + private readonly metrics: Metrics | null; + private readonly gossipQueues = createGossipQueues(); + private readonly gossipTopicConcurrency = mapValues(this.gossipQueues, () => 0); + private readonly extractBlockSlotRootFns = createExtractBlockSlotRootFns(); + // we may not receive the block for Attestation and SignedAggregateAndProof messages, in that case PendingGossipsubMessage needs + // to be stored in this Map and reprocessed once the block comes + private readonly awaitingGossipsubMessagesByRootBySlot: MapDef>>; + private unknownBlockGossipsubMessagesCount = 0; + + constructor(modules: NetworkProcessorModules, private readonly opts: NetworkProcessorOpts) { + const {chain, events, logger, metrics} = modules; + this.chain = chain; + this.events = events; + this.metrics = metrics; + this.logger = logger; + this.worker = new NetworkWorker(modules, opts); + + events.on(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage.bind(this)); + this.chain.emitter.on(routes.events.EventType.block, this.onBlockProcessed.bind(this)); + this.chain.emitter.on(ChainEvent.clockSlot, this.onClockSlot.bind(this)); + + this.awaitingGossipsubMessagesByRootBySlot = new MapDef( + () => new MapDef>(() => new Set()) + ); + + if (metrics) { + metrics.gossipValidationQueueLength.addCollect(() => { + for (const topic of executeGossipWorkOrder) { + metrics.gossipValidationQueueLength.set({topic}, this.gossipQueues[topic].length); + metrics.gossipValidationQueueConcurrency.set({topic}, this.gossipTopicConcurrency[topic]); + } + metrics.reprocessGossipAttestations.countPerSlot.set(this.unknownBlockGossipsubMessagesCount); + }); + } + + // TODO: Pull new work when available + // this.bls.onAvailable(() => this.executeWork()); + // this.regen.onAvailable(() => this.executeWork()); + } + + async stop(): Promise { + this.events.off(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage); + this.chain.emitter.off(routes.events.EventType.block, this.onBlockProcessed); + this.chain.emitter.off(ChainEvent.clockSlot, this.onClockSlot); + } + + dropAllJobs(): void { + for (const topic of executeGossipWorkOrder) { + this.gossipQueues[topic].clear(); + } + } + + dumpGossipQueue(topic: GossipType): PendingGossipsubMessage[] { + const queue = this.gossipQueues[topic]; + if (queue === undefined) { + throw Error(`Unknown gossipType ${topic}, known values: ${Object.keys(this.gossipQueues).join(", ")}`); + } + + return queue.getAll(); + } + + private onPendingGossipsubMessage(message: PendingGossipsubMessage): void { + const extractBlockSlotRootFn = this.extractBlockSlotRootFns[message.topic.type]; + // check block root of Attestation and SignedAggregateAndProof messages + if (extractBlockSlotRootFn) { + const slotRoot = extractBlockSlotRootFn(message.msg.data); + // if slotRoot is null, it means the msg.data is invalid + // in that case message will be rejected when deserializing data in later phase (gossipValidatorFn) + if (slotRoot && !this.chain.forkChoice.hasBlockHex(slotRoot.root)) { + if (this.unknownBlockGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) { + // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache + this.metrics?.reprocessGossipAttestations.reject.inc({reason: ReprocessRejectReason.reached_limit}); + return; + } + + this.metrics?.reprocessGossipAttestations.total.inc(); + const awaitingGossipsubMessagesByRoot = this.awaitingGossipsubMessagesByRootBySlot.getOrDefault(slotRoot.slot); + const awaitingGossipsubMessages = awaitingGossipsubMessagesByRoot.getOrDefault(slotRoot.root); + awaitingGossipsubMessages.add(message); + this.unknownBlockGossipsubMessagesCount++; + } + } + + // bypass the check for other messages + this.pushPendingGossipsubMessageToQueue(message); + } + + private pushPendingGossipsubMessageToQueue(message: PendingGossipsubMessage): void { + const topicType = message.topic.type; + const droppedJob = this.gossipQueues[topicType].add(message); + if (droppedJob) { + // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache + this.metrics?.gossipValidationQueueDroppedJobs.inc({topic: message.topic.type}); + } + + // Tentatively perform work + this.executeWork(); + } + + private async onBlockProcessed({ + slot, + block: rootHex, + }: { + slot: Slot; + block: string; + executionOptimistic: boolean; + }): Promise { + const byRootGossipsubMessages = this.awaitingGossipsubMessagesByRootBySlot.getOrDefault(slot); + const waitingGossipsubMessages = byRootGossipsubMessages.getOrDefault(rootHex); + if (waitingGossipsubMessages.size === 0) { + return; + } + + this.metrics?.reprocessGossipAttestations.resolve.inc(waitingGossipsubMessages.size); + const nowSec = Date.now() / 1000; + let count = 0; + // TODO: we can group attestations to process in batches but since we have the SeenAttestationDatas + // cache, it may not be necessary at this time + for (const message of waitingGossipsubMessages) { + this.metrics?.reprocessGossipAttestations.waitSecBeforeResolve.set(nowSec - message.seenTimestampSec); + this.pushPendingGossipsubMessageToQueue(message); + count++; + // don't want to block the event loop, worse case it'd wait for 16_084 / 1024 * 50ms = 800ms which is not a big deal + if (count === MAX_UNKNOWN_BLOCK_GOSSIP_OBJECTS_PER_TICK) { + count = 0; + await sleep(PROCESS_UNKNOWN_BLOCK_GOSSIP_OBJECTS_YIELD_EVERY_MS); + } + } + + byRootGossipsubMessages.delete(rootHex); + } + + private onClockSlot(clockSlot: Slot): void { + const nowSec = Date.now() / 1000; + for (const [slot, gossipMessagesByRoot] of this.awaitingGossipsubMessagesByRootBySlot.entries()) { + if (slot < clockSlot) { + for (const gossipMessages of gossipMessagesByRoot.values()) { + for (const message of gossipMessages) { + this.metrics?.reprocessGossipAttestations.reject.inc({reason: ReprocessRejectReason.expired}); + this.metrics?.reprocessGossipAttestations.waitSecBeforeReject.set(nowSec - message.seenTimestampSec); + // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache + } + } + this.awaitingGossipsubMessagesByRootBySlot.delete(slot); + } + } + this.unknownBlockGossipsubMessagesCount = 0; + } + + private executeWork(): void { + // TODO: Maybe de-bounce by timing the last time executeWork was run + + this.metrics?.networkProcessor.executeWorkCalls.inc(); + let jobsSubmitted = 0; + + job_loop: while (jobsSubmitted < MAX_JOBS_SUBMITTED_PER_TICK) { + // Check canAcceptWork before calling queue.next() since it consumes the items + const canAcceptWork = this.chain.blsThreadPoolCanAcceptWork() && this.chain.regenCanAcceptWork(); + + for (const topic of executeGossipWorkOrder) { + // beacon block is guaranteed to be processed immedately + if (!canAcceptWork && !executeGossipWorkOrderObj[topic]?.bypassQueue) { + this.metrics?.networkProcessor.canNotAcceptWork.inc(); + break job_loop; + } + if ( + this.opts.maxGossipTopicConcurrency !== undefined && + this.gossipTopicConcurrency[topic] > this.opts.maxGossipTopicConcurrency + ) { + // Reached concurrency limit for topic, continue to next topic + continue; + } + + const item = this.gossipQueues[topic].next(); + if (item) { + this.gossipTopicConcurrency[topic]++; + this.worker + .processPendingGossipsubMessage(item) + .finally(() => this.gossipTopicConcurrency[topic]--) + .catch((e) => this.logger.error("processGossipAttestations must not throw", {}, e)); + + jobsSubmitted++; + // Attempt to find more work, but check canAcceptWork() again and run executeGossipWorkOrder priorization + continue job_loop; + } + } + + // No item of work available on all queues, break off job_loop + break; + } + + this.metrics?.networkProcessor.jobsSubmitted.observe(jobsSubmitted); + } +} diff --git a/packages/beacon-node/src/network/processor/types.ts b/packages/beacon-node/src/network/processor/types.ts new file mode 100644 index 000000000000..660986ead1c0 --- /dev/null +++ b/packages/beacon-node/src/network/processor/types.ts @@ -0,0 +1,22 @@ +import {PeerId} from "@libp2p/interface-peer-id"; +import {Message} from "@libp2p/interface-pubsub"; +import {SlotRootHex} from "@lodestar/types"; +import {GossipTopic, GossipType} from "../gossip/index.js"; + +export type GossipAttestationsWork = { + messages: PendingGossipsubMessage[]; +}; + +export type PendingGossipsubMessage = { + topic: GossipTopic; + msg: Message; + msgId: string; + // TODO: Refactor into accepting string (requires gossipsub changes) for easier multi-threading + propagationSource: PeerId; + seenTimestampSec: number; + startProcessUnixSec: number | null; +}; + +export type ExtractSlotRootFns = { + [K in GossipType]?: (data: Uint8Array) => SlotRootHex | null; +}; diff --git a/packages/beacon-node/src/network/processor/worker.ts b/packages/beacon-node/src/network/processor/worker.ts new file mode 100644 index 000000000000..dc8fb82a6d34 --- /dev/null +++ b/packages/beacon-node/src/network/processor/worker.ts @@ -0,0 +1,64 @@ +import {IBeaconChain} from "../../chain/interface.js"; +import {Metrics} from "../../metrics/metrics.js"; +import {NetworkEvent, NetworkEventBus} from "../events.js"; +import {GossipHandlers, GossipValidatorFn} from "../gossip/interface.js"; +import {getGossipHandlers, GossipHandlerOpts, ValidatorFnsModules} from "./gossipHandlers.js"; +import {getGossipValidatorFn, ValidatorFnModules} from "./gossipValidatorFn.js"; +import {PendingGossipsubMessage} from "./types.js"; + +export type NetworkWorkerModules = ValidatorFnsModules & + ValidatorFnModules & { + chain: IBeaconChain; + events: NetworkEventBus; + metrics: Metrics | null; + // Optionally pass custom GossipHandlers, for testing + gossipHandlers?: GossipHandlers; + }; + +export class NetworkWorker { + private readonly events: NetworkEventBus; + private readonly metrics: Metrics | null; + private readonly gossipValidatorFn: GossipValidatorFn; + + constructor(modules: NetworkWorkerModules, opts: GossipHandlerOpts) { + this.events = modules.events; + this.metrics = modules.metrics; + this.gossipValidatorFn = getGossipValidatorFn(modules.gossipHandlers ?? getGossipHandlers(modules, opts), modules); + } + + async processPendingGossipsubMessage(message: PendingGossipsubMessage): Promise { + message.startProcessUnixSec = Date.now() / 1000; + + const acceptance = await this.gossipValidatorFn( + message.topic, + message.msg, + message.propagationSource.toString(), + message.seenTimestampSec + ); + + if (message.startProcessUnixSec !== null) { + this.metrics?.gossipValidationQueueJobWaitTime.observe( + {topic: message.topic.type}, + message.startProcessUnixSec - message.seenTimestampSec + ); + this.metrics?.gossipValidationQueueJobTime.observe( + {topic: message.topic.type}, + Date.now() / 1000 - message.startProcessUnixSec + ); + } + + // Use setTimeout to yield to the macro queue + // This is mostly due to too many attestation messages, and a gossipsub RPC may + // contain multiple of them. This helps avoid the I/O lag issue. + setTimeout( + () => + this.events.emit( + NetworkEvent.gossipMessageValidationResult, + message.msgId, + message.propagationSource, + acceptance + ), + 0 + ); + } +} diff --git a/packages/beacon-node/src/util/queue/itemQueue.ts b/packages/beacon-node/src/util/queue/itemQueue.ts index 802a9b0e84ec..46bb2b62f55b 100644 --- a/packages/beacon-node/src/util/queue/itemQueue.ts +++ b/packages/beacon-node/src/util/queue/itemQueue.ts @@ -41,6 +41,10 @@ export class JobItemQueue { } } + get jobLen(): number { + return this.jobs.length; + } + push(...args: Args): Promise { if (this.opts.signal.aborted) { throw new QueueError({code: QueueErrorCode.QUEUE_ABORTED}); diff --git a/packages/beacon-node/src/util/sszBytes.ts b/packages/beacon-node/src/util/sszBytes.ts new file mode 100644 index 000000000000..a0ffff0ceca8 --- /dev/null +++ b/packages/beacon-node/src/util/sszBytes.ts @@ -0,0 +1,127 @@ +import {RootHex, Slot} from "@lodestar/types"; +import {toHex} from "@lodestar/utils"; + +export type BlockRootHex = RootHex; +export type AttDataBase64 = string; + +// class Attestation(Container): +// aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] - offset 4 +// data: AttestationData - target data +// signature: BLSSignature +// +// class AttestationData(Container): 128 bytes fixed size +// slot: Slot - data 8 +// index: CommitteeIndex - data 8 +// beacon_block_root: Root - data 32 +// source: Checkpoint - data 40 +// target: Checkpoint - data 40 +// +// class SignedAggregateAndProof(Container): +// message: AggregateAndProof - offset 4 +// signature: BLSSignature - data 96 + +// class AggregateAndProof(Container) +// aggregatorIndex: ValidatorIndex - data 8 +// aggregate: Attestation - offset 4 +// selectionProof: BLSSignature - data 96 + +const ATTESTATION_SLOT_OFFSET = 4; +const ATTESTATION_BEACON_BLOCK_ROOT_OFFSET = ATTESTATION_SLOT_OFFSET + 8 + 8; +const ROOT_SIZE = 32; +const SLOT_SIZE = 8; +const ATTESTATION_DATA_SIZE = 128; + +/** + * Extract slot from attestation serialized bytes. + * Return null if data is not long enough to extract slot. + */ +export function getSlotFromAttestationSerialized(data: Uint8Array): Slot | null { + if (data.length < ATTESTATION_SLOT_OFFSET + SLOT_SIZE) { + return null; + } + + return getSlotFromOffset(data, ATTESTATION_SLOT_OFFSET); +} + +/** + * Extract block root from attestation serialized bytes. + * Return null if data is not long enough to extract block root. + */ +export function getBlockRootFromAttestationSerialized(data: Uint8Array): BlockRootHex | null { + if (data.length < ATTESTATION_BEACON_BLOCK_ROOT_OFFSET + ROOT_SIZE) { + return null; + } + + return toHex(data.subarray(ATTESTATION_BEACON_BLOCK_ROOT_OFFSET, ATTESTATION_BEACON_BLOCK_ROOT_OFFSET + ROOT_SIZE)); +} + +/** + * Extract attestation data base64 from attestation serialized bytes. + * Return null if data is not long enough to extract attestation data. + */ +export function getAttDataBase64FromAttestationSerialized(data: Uint8Array): AttDataBase64 | null { + if (data.length < ATTESTATION_SLOT_OFFSET + ATTESTATION_DATA_SIZE) { + return null; + } + + // base64 is a bit efficient than hex + return Buffer.from(data.slice(ATTESTATION_SLOT_OFFSET, ATTESTATION_SLOT_OFFSET + ATTESTATION_DATA_SIZE)).toString( + "base64" + ); +} + +const AGGREGATE_AND_PROOF_OFFSET = 4 + 96; +const AGGREGATE_OFFSET = AGGREGATE_AND_PROOF_OFFSET + 8 + 4 + 96; +const SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET = AGGREGATE_OFFSET + ATTESTATION_SLOT_OFFSET; +const SIGNED_AGGREGATE_AND_PROOF_BLOCK_ROOT_OFFSET = SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET + 8 + 8; + +/** + * Extract slot from signed aggregate and proof serialized bytes. + * Return null if data is not long enough to extract slot. + */ +export function getSlotFromSignedAggregateAndProofSerialized(data: Uint8Array): Slot | null { + if (data.length < SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET + SLOT_SIZE) { + return null; + } + + return getSlotFromOffset(data, SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET); +} + +/** + * Extract block root from signed aggregate and proof serialized bytes. + * Return null if data is not long enough to extract block root. + */ +export function getBlockRootFromSignedAggregateAndProofSerialized(data: Uint8Array): BlockRootHex | null { + if (data.length < SIGNED_AGGREGATE_AND_PROOF_BLOCK_ROOT_OFFSET + ROOT_SIZE) { + return null; + } + + return toHex( + data.subarray( + SIGNED_AGGREGATE_AND_PROOF_BLOCK_ROOT_OFFSET, + SIGNED_AGGREGATE_AND_PROOF_BLOCK_ROOT_OFFSET + ROOT_SIZE + ) + ); +} + +/** + * Extract attestation data base64 from signed aggregate and proof serialized bytes. + * Return null if data is not long enough to extract attestation data. + */ +export function getAttDataBase64FromSignedAggregateAndProofSerialized(data: Uint8Array): AttDataBase64 | null { + if (data.length < SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET + ATTESTATION_DATA_SIZE) { + return null; + } + + // base64 is a bit efficient than hex + return Buffer.from( + data.slice(SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET, SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET + ATTESTATION_DATA_SIZE) + ).toString("base64"); +} + +function getSlotFromOffset(data: Uint8Array, offset: number): Slot { + // TODO: Optimize + const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); + // Read only the first 4 bytes of Slot, max value is 4,294,967,295 will be reached 1634 years after genesis + return dv.getUint32(offset, true); +} diff --git a/packages/beacon-node/test/e2e/network/gossipsub.test.ts b/packages/beacon-node/test/e2e/network/gossipsub.test.ts index ae80c66d9a2b..ee1bb71b2a75 100644 --- a/packages/beacon-node/test/e2e/network/gossipsub.test.ts +++ b/packages/beacon-node/test/e2e/network/gossipsub.test.ts @@ -5,7 +5,7 @@ import {capella, phase0, ssz, allForks} from "@lodestar/types"; import {sleep} from "@lodestar/utils"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; -import {getReqRespHandlers, Network} from "../../../src/network/index.js"; +import {getReqRespHandlers, Network, NetworkInitModules} from "../../../src/network/index.js"; import {defaultNetworkOptions, NetworkOptions} from "../../../src/network/options.js"; import {GossipType, GossipHandlers} from "../../../src/network/gossip/index.js"; @@ -25,6 +25,7 @@ const opts: NetworkOptions = { localMultiaddrs: [], discv5FirstQueryDelayMs: 0, discv5: null, + skipParamsLog: true, }; // Schedule all forks at ALTAIR_FORK_EPOCH to avoid generating the pubkeys cache @@ -86,10 +87,9 @@ describe("gossipsub", function () { const loggerA = testLogger("A"); const loggerB = testLogger("B"); - const modules = { + const modules: Omit = { config: beaconConfig, chain, - db, reqRespHandlers, gossipHandlers, signal: controller.signal, diff --git a/packages/beacon-node/test/memory/seenAttestationData.ts b/packages/beacon-node/test/memory/seenAttestationData.ts new file mode 100644 index 000000000000..c6735bd861e9 --- /dev/null +++ b/packages/beacon-node/test/memory/seenAttestationData.ts @@ -0,0 +1,59 @@ +import crypto from "node:crypto"; +import {toHexString} from "@chainsafe/ssz"; +import {AttestationDataCacheEntry, SeenAttestationDatas} from "../../src/chain/seenCache/seenAttestationData.js"; +import {testRunnerMemory} from "./testRunnerMemory.js"; + +/** + * SeenAttestationDatas 64 keys - 88039.8 bytes / instance + * SeenAttestationDatas 128 keys - 177436.8 bytes / instance + * SeenAttestationDatas 200 keys - 276592.0 bytes / instance + */ +testRunnerMemoryBpi([ + { + id: "SeenAttestationDatas 64 keys", + getInstance: () => getRandomSeenAttestationDatas(64), + }, + { + id: "SeenAttestationDatas 128 keys", + getInstance: () => getRandomSeenAttestationDatas(128), + }, + { + id: "SeenAttestationDatas 200 keys", + getInstance: () => getRandomSeenAttestationDatas(200), + }, +]); + +function getRandomSeenAttestationDatas(n: number): SeenAttestationDatas { + const seenAttestationDatas = new SeenAttestationDatas(null); + const slot = 1000; + for (let i = 0; i < n; i++) { + const attDataBytes = crypto.randomBytes(128); + const key = Buffer.from(attDataBytes).toString("base64"); + // skip index2pubkey and committeeIndices as they are shared + const attDataCacheEntry = { + signingRoot: crypto.randomBytes(32), + attDataRootHex: toHexString(crypto.randomBytes(32)), + subnet: i, + } as unknown as AttestationDataCacheEntry; + seenAttestationDatas.add(slot, key, attDataCacheEntry); + } + return seenAttestationDatas; +} + +/** + * Test bytes per instance in different representations of raw binary data + */ +function testRunnerMemoryBpi(testCases: {getInstance: (bytes: number) => unknown; id: string}[]): void { + const longestId = Math.max(...testCases.map(({id}) => id.length)); + + for (const {id, getInstance} of testCases) { + const bpi = testRunnerMemory({ + getInstance, + convergeFactor: 1 / 100, + sampleEvery: 5, + }); + + // eslint-disable-next-line no-console + console.log(`${id.padEnd(longestId)} - ${bpi.toFixed(1)} bytes / instance`); + } +} diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index e4f202684a29..77c7bcd9948c 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -10,6 +10,7 @@ import { import {HISTORICAL_ROOTS_LIMIT, SLOTS_PER_EPOCH, TIMELY_SOURCE_FLAG_INDEX} from "@lodestar/params"; import {BitArray, toHexString} from "@chainsafe/ssz"; import {ExecutionStatus, ForkChoice, IForkChoiceStore, ProtoArray} from "@lodestar/fork-choice"; +import {ssz} from "@lodestar/types"; import {AggregatedAttestationPool} from "../../../../src/chain/opPools/aggregatedAttestationPool.js"; import {generatePerfTestCachedStateAltair} from "../../../../../state-transition/test/perf/util.js"; import {computeAnchorCheckpoint} from "../../../../src/chain/initState.js"; @@ -162,7 +163,12 @@ function getAggregatedAttestationPool(state: CachedBeaconStateAltair): Aggregate const committee = state.epochCtx.getBeaconCommittee(slot, committeeIndex); // all attestation has full participation so getAttestationsForBlock() has to do a lot of filter // aggregate_and_proof messages - pool.add(attestation, committee.length, committee); + pool.add( + attestation, + toHexString(ssz.phase0.AttestationData.hashTreeRoot(attestation.data)), + committee.length, + committee + ); } } return pool; diff --git a/packages/beacon-node/test/spec/presets/fork_choice.ts b/packages/beacon-node/test/spec/presets/fork_choice.ts index d8be41ffe703..7bdaa95b0139 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.ts @@ -118,7 +118,8 @@ export const forkChoiceTest = const attestation = testcase.attestations.get(step.attestation); if (!attestation) throw Error(`No attestation ${step.attestation}`); const headState = chain.getHeadState(); - chain.forkChoice.onAttestation(headState.epochCtx.getIndexedAttestation(attestation)); + const attDataRootHex = toHexString(ssz.phase0.AttestationData.hashTreeRoot(attestation.data)); + chain.forkChoice.onAttestation(headState.epochCtx.getIndexedAttestation(attestation), attDataRootHex); } // attester slashing step diff --git a/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts b/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts index bd79274b8a64..06bec4d61b2f 100644 --- a/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts +++ b/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts @@ -110,9 +110,9 @@ describe("LodestarForkChoice", function () { const attestation0 = createIndexedAttestation(source, targetBlock, orphanedBlock, 0); const attestation1 = createIndexedAttestation(source, targetBlock, parentBlock, 1); const attestation2 = createIndexedAttestation(source, targetBlock, childBlock, 2); - forkChoice.onAttestation(attestation0); - forkChoice.onAttestation(attestation1); - forkChoice.onAttestation(attestation2); + forkChoice.onAttestation(attestation0, toHexString(ssz.phase0.AttestationData.hashTreeRoot(attestation0.data))); + forkChoice.onAttestation(attestation1, toHexString(ssz.phase0.AttestationData.hashTreeRoot(attestation1.data))); + forkChoice.onAttestation(attestation2, toHexString(ssz.phase0.AttestationData.hashTreeRoot(attestation2.data))); head = forkChoice.getHead(); // with votes, head becomes the child block expect(head.slot).to.be.equal(childBlock.message.slot); @@ -284,7 +284,7 @@ describe("LodestarForkChoice", function () { }, signature: Buffer.alloc(96), }; - forkChoice.onAttestation(attestation); + forkChoice.onAttestation(attestation, toHexString(ssz.phase0.AttestationData.hashTreeRoot(attestation.data))); } // Z stays on next epoch, a child of X which potentially reorg Y diff --git a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts index f4c7541e78fc..3f2bd166c417 100644 --- a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts @@ -3,7 +3,7 @@ import {SinonStubbedInstance} from "sinon"; import sinon from "sinon"; import type {SecretKey} from "@chainsafe/bls/types"; import bls from "@chainsafe/bls"; -import {BitArray, fromHexString} from "@chainsafe/ssz"; +import {BitArray, fromHexString, toHexString} from "@chainsafe/ssz"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {ssz, phase0} from "@lodestar/types"; @@ -37,6 +37,7 @@ describe("AggregatedAttestationPool", function () { const attestation = ssz.phase0.Attestation.defaultValue(); attestation.data.slot = currentSlot; attestation.data.target.epoch = currentEpoch; + const attDataRootHex = toHexString(ssz.phase0.AttestationData.hashTreeRoot(attestation.data)); const committee = [0, 1, 2, 3]; let forkchoiceStub: SinonStubbedInstance; @@ -71,7 +72,12 @@ describe("AggregatedAttestationPool", function () { for (const {name, attestingBits, isReturned} of testCases) { it(name, function () { const aggregationBits = new BitArray(new Uint8Array(attestingBits), 8); - pool.add({...attestation, aggregationBits}, aggregationBits.getTrueBitIndexes().length, committee); + pool.add( + {...attestation, aggregationBits}, + attDataRootHex, + aggregationBits.getTrueBitIndexes().length, + committee + ); forkchoiceStub.getBlockHex.returns(generateProtoBlock()); forkchoiceStub.getDependentRoot.returns(ZERO_HASH_HEX); if (isReturned) { @@ -90,7 +96,7 @@ describe("AggregatedAttestationPool", function () { altairState.currentJustifiedCheckpoint.epoch = 1000; // all attesters are not seen const attestingIndices = [2, 3]; - pool.add(attestation, attestingIndices.length, committee); + pool.add(attestation, attDataRootHex, attestingIndices.length, committee); expect(pool.getAttestationsForBlock(forkchoiceStub, altairState)).to.be.deep.equal( [], "no attestation since incorrect source" @@ -101,7 +107,7 @@ describe("AggregatedAttestationPool", function () { it("incompatible shuffling - incorrect pivot block root", function () { // all attesters are not seen const attestingIndices = [2, 3]; - pool.add(attestation, attestingIndices.length, committee); + pool.add(attestation, attDataRootHex, attestingIndices.length, committee); forkchoiceStub.getBlockHex.returns(generateProtoBlock()); forkchoiceStub.getDependentRoot.returns("0xWeird"); expect(pool.getAttestationsForBlock(forkchoiceStub, altairState)).to.be.deep.equal( diff --git a/packages/beacon-node/test/unit/chain/seenCache/seenAttestationData.test.ts b/packages/beacon-node/test/unit/chain/seenCache/seenAttestationData.test.ts new file mode 100644 index 000000000000..12d960c4b89c --- /dev/null +++ b/packages/beacon-node/test/unit/chain/seenCache/seenAttestationData.test.ts @@ -0,0 +1,53 @@ +import {expect} from "chai"; +import {InsertOutcome} from "../../../../src/chain/opPools/types.js"; +import {AttestationDataCacheEntry, SeenAttestationDatas} from "../../../../src/chain/seenCache/seenAttestationData.js"; + +// Compare this snippet from packages/beacon-node/src/chain/seenCache/seenAttestationData.ts: +describe("SeenAttestationDatas", () => { + // Only accept AttestationData from current slot or previous slot + // Max cache size per slot is 2 + let cache: SeenAttestationDatas; + + beforeEach(() => { + cache = new SeenAttestationDatas(null, 1, 2); + cache.onSlot(100); + cache.add(99, "99a", {attDataRootHex: "99a"} as AttestationDataCacheEntry); + cache.add(99, "99b", {attDataRootHex: "99b"} as AttestationDataCacheEntry); + cache.add(100, "100a", {attDataRootHex: "100a"} as AttestationDataCacheEntry); + }); + + const addTestCases: {slot: number; attDataBase64: string; expected: InsertOutcome}[] = [ + {slot: 98, attDataBase64: "98a", expected: InsertOutcome.Old}, + {slot: 99, attDataBase64: "99a", expected: InsertOutcome.AlreadyKnown}, + {slot: 99, attDataBase64: "99c", expected: InsertOutcome.ReachLimit}, + {slot: 100, attDataBase64: "100b", expected: InsertOutcome.NewData}, + ]; + + for (const testCase of addTestCases) { + it(`add slot ${testCase.slot} data ${testCase.attDataBase64} should return ${testCase.expected}`, () => { + expect( + cache.add(testCase.slot, testCase.attDataBase64, { + attDataRootHex: testCase.attDataBase64, + } as AttestationDataCacheEntry) + ).to.equal(testCase.expected); + }); + } + + const getTestCases: {slot: number; attDataBase64: string; expectedNull: boolean}[] = [ + {slot: 98, attDataBase64: "98a", expectedNull: true}, + {slot: 99, attDataBase64: "99unknown", expectedNull: true}, + {slot: 99, attDataBase64: "99a", expectedNull: false}, + ]; + + for (const testCase of getTestCases) { + it(`get slot ${testCase.slot} data ${testCase.attDataBase64} should return ${ + testCase.expectedNull ? "null" : "not null" + }`, () => { + if (testCase.expectedNull) { + expect(cache.get(testCase.slot, testCase.attDataBase64)).to.be.null; + } else { + expect(cache.get(testCase.slot, testCase.attDataBase64)).to.not.be.null; + } + }); + } +}); diff --git a/packages/beacon-node/test/unit/chain/validation/block.test.ts b/packages/beacon-node/test/unit/chain/validation/block.test.ts index 4bf1223e70f4..8c84ae3e1ee3 100644 --- a/packages/beacon-node/test/unit/chain/validation/block.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/block.test.ts @@ -44,7 +44,7 @@ describe("gossip block validation", function () { verifySignature = sinon.stub(); verifySignature.resolves(true); - chain.bls = {verifySignatureSets: verifySignature, close: () => Promise.resolve()}; + chain.bls = {verifySignatureSets: verifySignature, close: () => Promise.resolve(), canAcceptWork: () => true}; forkChoice.getFinalizedCheckpoint.returns({epoch: 0, root: ZERO_HASH, rootHex: ""}); diff --git a/packages/beacon-node/test/unit/network/processorQueues.test.ts b/packages/beacon-node/test/unit/network/processorQueues.test.ts new file mode 100644 index 000000000000..0c272159e51a --- /dev/null +++ b/packages/beacon-node/test/unit/network/processorQueues.test.ts @@ -0,0 +1,108 @@ +import {expect} from "chai"; +import {sleep} from "@lodestar/utils"; + +type ValidateOpts = { + skipAsync1: boolean; + skipAsync2: boolean; +}; + +async function validateTest(job: string, tracker: string[], opts: ValidateOpts): Promise { + tracker.push(`job:${job} step:0`); + + await getStateFromCache(opts.skipAsync1); + tracker.push(`job:${job} step:1`); + + if (!opts.skipAsync2) { + await sleep(0); + } + tracker.push(`job:${job} step:2`); +} + +async function getStateFromCache(retrieveSync: boolean): Promise { + if (retrieveSync) { + return 1; + } else { + await sleep(0); + return 2; + } +} + +describe("event loop with branching async", () => { + const eachAwaitPointHoldsJobs = [ + "job:0 step:0", + "job:1 step:0", + "job:2 step:0", + "job:0 step:1", + "job:1 step:1", + "job:2 step:1", + "job:0 step:2", + "job:1 step:2", + "job:2 step:2", + ]; + + const onlyStartOfStep1HoldsJobs = [ + "job:0 step:0", + "job:1 step:0", + "job:2 step:0", + "job:0 step:1", + "job:0 step:2", + "job:1 step:1", + "job:1 step:2", + "job:2 step:1", + "job:2 step:2", + ]; + + const eachJobCompletesInSequence = [ + "job:0 step:0", + "job:0 step:1", + "job:0 step:2", + "job:1 step:0", + "job:1 step:1", + "job:1 step:2", + "job:2 step:0", + "job:2 step:1", + "job:2 step:2", + ]; + + const testCases: {opts: ValidateOpts; expectedTrackerVoid: string[]; expectedTrackerAwait: string[]}[] = [ + { + opts: {skipAsync1: false, skipAsync2: false}, + expectedTrackerVoid: eachAwaitPointHoldsJobs, + expectedTrackerAwait: eachJobCompletesInSequence, + }, + { + opts: {skipAsync1: true, skipAsync2: false}, + expectedTrackerVoid: eachAwaitPointHoldsJobs, + expectedTrackerAwait: eachJobCompletesInSequence, + }, + { + opts: {skipAsync1: false, skipAsync2: true}, + expectedTrackerVoid: onlyStartOfStep1HoldsJobs, + expectedTrackerAwait: eachJobCompletesInSequence, + }, + { + opts: {skipAsync1: true, skipAsync2: true}, + expectedTrackerVoid: onlyStartOfStep1HoldsJobs, + expectedTrackerAwait: eachJobCompletesInSequence, + }, + ]; + + for (const {opts, expectedTrackerVoid, expectedTrackerAwait} of testCases) { + const jobs: string[] = []; + for (let i = 0; i < 3; i++) jobs.push(String(i)); + + it(`${JSON.stringify(opts)} Promise.all`, async () => { + const tracker: string[] = []; + await Promise.all(jobs.map((job) => validateTest(job, tracker, opts))); + expect(tracker).deep.equals(expectedTrackerVoid); + }); + + it(`${JSON.stringify(opts)} await each`, async () => { + const tracker: string[] = []; + for (const job of jobs) { + await validateTest(job, tracker, opts); + } + expect(tracker).deep.equals(expectedTrackerAwait); + }); + } +}); diff --git a/packages/beacon-node/test/unit/util/sszBytes.test.ts b/packages/beacon-node/test/unit/util/sszBytes.test.ts new file mode 100644 index 000000000000..06aa32979857 --- /dev/null +++ b/packages/beacon-node/test/unit/util/sszBytes.test.ts @@ -0,0 +1,137 @@ +import {expect} from "chai"; +import {Epoch, phase0, RootHex, Slot, ssz} from "@lodestar/types"; +import {fromHex, toHex} from "@lodestar/utils"; +import { + getAttDataBase64FromAttestationSerialized, + getAttDataBase64FromSignedAggregateAndProofSerialized, + getBlockRootFromAttestationSerialized, + getBlockRootFromSignedAggregateAndProofSerialized, + getSlotFromAttestationSerialized, + getSlotFromSignedAggregateAndProofSerialized, +} from "../../../src/util/sszBytes.js"; + +describe("attestation SSZ serialized picking", () => { + const testCases: phase0.Attestation[] = [ + ssz.phase0.Attestation.defaultValue(), + attestationFromValues( + 4_000_000, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + 200_00, + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffffffffffff" + ), + ]; + + for (const [i, attestation] of testCases.entries()) { + it(`attestation ${i}`, () => { + const bytes = ssz.phase0.Attestation.serialize(attestation); + + expect(getSlotFromAttestationSerialized(bytes)).equals(attestation.data.slot); + expect(getBlockRootFromAttestationSerialized(bytes)).equals(toHex(attestation.data.beaconBlockRoot)); + + const attDataBase64 = ssz.phase0.AttestationData.serialize(attestation.data); + expect(getAttDataBase64FromAttestationSerialized(bytes)).to.be.equal( + Buffer.from(attDataBase64).toString("base64") + ); + }); + } + + it("getSlotFromAttestationSerialized - invalid data", () => { + const invalidSlotDataSizes = [0, 4, 11]; + for (const size of invalidSlotDataSizes) { + expect(getSlotFromAttestationSerialized(Buffer.alloc(size))).to.be.null; + } + }); + + it("getBlockRootFromAttestationSerialized - invalid data", () => { + const invalidBlockRootDataSizes = [0, 4, 20, 49]; + for (const size of invalidBlockRootDataSizes) { + expect(getBlockRootFromAttestationSerialized(Buffer.alloc(size))).to.be.null; + } + }); + + it("getAttDataBase64FromAttestationSerialized - invalid data", () => { + const invalidAttDataBase64DataSizes = [0, 4, 100, 128, 131]; + for (const size of invalidAttDataBase64DataSizes) { + expect(getAttDataBase64FromAttestationSerialized(Buffer.alloc(size))).to.be.null; + } + }); +}); + +describe("aggregateAndProof SSZ serialized peaking", () => { + const testCases: phase0.SignedAggregateAndProof[] = [ + ssz.phase0.SignedAggregateAndProof.defaultValue(), + signedAggregateAndProofFromValues( + 4_000_000, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + 200_00, + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffffffffffff" + ), + ]; + + for (const [i, signedAggregateAndProof] of testCases.entries()) { + it(`signedAggregateAndProof ${i}`, () => { + const bytes = ssz.phase0.SignedAggregateAndProof.serialize(signedAggregateAndProof); + + expect(getSlotFromSignedAggregateAndProofSerialized(bytes)).equals( + signedAggregateAndProof.message.aggregate.data.slot + ); + expect(getBlockRootFromSignedAggregateAndProofSerialized(bytes)).equals( + toHex(signedAggregateAndProof.message.aggregate.data.beaconBlockRoot) + ); + + const attDataBase64 = ssz.phase0.AttestationData.serialize(signedAggregateAndProof.message.aggregate.data); + expect(getAttDataBase64FromSignedAggregateAndProofSerialized(bytes)).to.be.equal( + Buffer.from(attDataBase64).toString("base64") + ); + }); + } + + it("getSlotFromSignedAggregateAndProofSerialized - invalid data", () => { + const invalidSlotDataSizes = [0, 4, 11]; + for (const size of invalidSlotDataSizes) { + expect(getSlotFromSignedAggregateAndProofSerialized(Buffer.alloc(size))).to.be.null; + } + }); + + it("getBlockRootFromSignedAggregateAndProofSerialized - invalid data", () => { + const invalidBlockRootDataSizes = [0, 4, 20, 227]; + for (const size of invalidBlockRootDataSizes) { + expect(getBlockRootFromSignedAggregateAndProofSerialized(Buffer.alloc(size))).to.be.null; + } + }); + + it("getAttDataBase64FromSignedAggregateAndProofSerialized - invalid data", () => { + const invalidAttDataBase64DataSizes = [0, 4, 100, 128, 339]; + for (const size of invalidAttDataBase64DataSizes) { + expect(getAttDataBase64FromSignedAggregateAndProofSerialized(Buffer.alloc(size))).to.be.null; + } + }); +}); + +function attestationFromValues( + slot: Slot, + blockRoot: RootHex, + targetEpoch: Epoch, + targetRoot: RootHex +): phase0.Attestation { + const attestation = ssz.phase0.Attestation.defaultValue(); + attestation.data.slot = slot; + attestation.data.beaconBlockRoot = fromHex(blockRoot); + attestation.data.target.epoch = targetEpoch; + attestation.data.target.root = fromHex(targetRoot); + return attestation; +} + +function signedAggregateAndProofFromValues( + slot: Slot, + blockRoot: RootHex, + targetEpoch: Epoch, + targetRoot: RootHex +): phase0.SignedAggregateAndProof { + const signedAggregateAndProof = ssz.phase0.SignedAggregateAndProof.defaultValue(); + signedAggregateAndProof.message.aggregate.data.slot = slot; + signedAggregateAndProof.message.aggregate.data.beaconBlockRoot = fromHex(blockRoot); + signedAggregateAndProof.message.aggregate.data.target.epoch = targetEpoch; + signedAggregateAndProof.message.aggregate.data.target.root = fromHex(targetRoot); + return signedAggregateAndProof; +} diff --git a/packages/beacon-node/test/utils/mocks/bls.ts b/packages/beacon-node/test/utils/mocks/bls.ts index 0013a2d49ead..57e84d509fc7 100644 --- a/packages/beacon-node/test/utils/mocks/bls.ts +++ b/packages/beacon-node/test/utils/mocks/bls.ts @@ -10,4 +10,8 @@ export class BlsVerifierMock implements IBlsVerifier { async close(): Promise { // } + + canAcceptWork(): boolean { + return true; + } } diff --git a/packages/beacon-node/test/utils/mocks/chain/chain.ts b/packages/beacon-node/test/utils/mocks/chain/chain.ts index a73978029e0e..45137f5c251e 100644 --- a/packages/beacon-node/test/utils/mocks/chain/chain.ts +++ b/packages/beacon-node/test/utils/mocks/chain/chain.ts @@ -3,7 +3,12 @@ import sinon from "sinon"; import {CompositeTypeAny, toHexString, TreeView} from "@chainsafe/ssz"; import {phase0, allForks, UintNum64, Root, Slot, ssz, Uint16, UintBn64, RootHex, deneb, Wei} from "@lodestar/types"; import {BeaconConfig} from "@lodestar/config"; -import {BeaconStateAllForks, CachedBeaconStateAllForks} from "@lodestar/state-transition"; +import { + BeaconStateAllForks, + CachedBeaconStateAllForks, + Index2PubkeyCache, + PubkeyIndexMap, +} from "@lodestar/state-transition"; import {CheckpointWithHex, IForkChoice, ProtoBlock, ExecutionStatus, AncestorStatus} from "@lodestar/fork-choice"; import {defaultOptions as defaultValidatorOptions} from "@lodestar/validator"; import {Logger} from "@lodestar/utils"; @@ -42,6 +47,7 @@ import {CheckpointBalancesCache} from "../../../../src/chain/balancesCache.js"; import {IChainOptions} from "../../../../src/chain/options.js"; import {BlockAttributes} from "../../../../src/chain/produceBlock/produceBlockBody.js"; import {ReqRespBlockResponse} from "../../../../src/network/index.js"; +import {SeenAttestationDatas} from "../../../../src/chain/seenCache/seenAttestationData.js"; /* eslint-disable @typescript-eslint/no-empty-function */ @@ -81,6 +87,8 @@ export class MockBeaconChain implements IBeaconChain { emitter: ChainEventEmitter; lightClientServer: LightClientServer; reprocessController: ReprocessController; + readonly pubkey2index: PubkeyIndexMap; + readonly index2pubkey: Index2PubkeyCache; // Ops pool readonly attestationPool: AttestationPool; @@ -96,6 +104,7 @@ export class MockBeaconChain implements IBeaconChain { readonly seenBlockProposers = new SeenBlockProposers(); readonly seenSyncCommitteeMessages = new SeenSyncCommitteeMessages(); readonly seenContributionAndProof = new SeenContributionAndProof(null); + readonly seenAttestationDatas = new SeenAttestationDatas(null); readonly seenBlockAttesters = new SeenBlockAttesters(); readonly beaconProposerCache = new BeaconProposerCache({ @@ -152,6 +161,8 @@ export class MockBeaconChain implements IBeaconChain { } ); this.reprocessController = new ReprocessController(null); + this.pubkey2index = new PubkeyIndexMap(); + this.index2pubkey = []; } validatorSeenAtEpoch(): boolean { @@ -227,6 +238,14 @@ export class MockBeaconChain implements IBeaconChain { async updateBeaconProposerData(): Promise {} updateBuilderStatus(): void {} + + regenCanAcceptWork(): boolean { + return true; + } + + blsThreadPoolCanAcceptWork(): boolean { + return true; + } } const root = ssz.Root.defaultValue() as Uint8Array; diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index 91a1b9e23070..20040ede6b1e 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -122,6 +122,7 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { seenAggregatedAttestations: new SeenAggregatedAttestations(null), bls: new BlsSingleThreadVerifier({metrics: null}), waitForBlock: () => Promise.resolve(false), + index2pubkey: state.epochCtx.index2pubkey, } as Partial as IBeaconChain; return {chain, attestation, subnet, validatorIndex}; diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts index 53f735dbaa50..4963348f2e15 100644 --- a/packages/cli/src/options/beaconNodeOptions/chain.ts +++ b/packages/cli/src/options/beaconNodeOptions/chain.ts @@ -13,6 +13,7 @@ export type ChainArgs = { "chain.proposerBoostEnabled": boolean; "chain.disableImportExecutionFcU": boolean; "chain.preaggregateSlotDistance": number; + "chain.attDataCacheSlotDistance": number; "chain.computeUnrealized": boolean; "chain.assertCorrectProgressiveBalances": boolean; "chain.maxSkipSlots": number; @@ -33,6 +34,7 @@ export function parseArgs(args: ChainArgs): IBeaconNodeOptions["chain"] { proposerBoostEnabled: args["chain.proposerBoostEnabled"], disableImportExecutionFcU: args["chain.disableImportExecutionFcU"], preaggregateSlotDistance: args["chain.preaggregateSlotDistance"], + attDataCacheSlotDistance: args["chain.attDataCacheSlotDistance"], computeUnrealized: args["chain.computeUnrealized"], assertCorrectProgressiveBalances: args["chain.assertCorrectProgressiveBalances"], maxSkipSlots: args["chain.maxSkipSlots"], @@ -113,6 +115,13 @@ Will double processing times. Use only for debugging purposes.", group: "chain", }, + "chain.attDataCacheSlotDistance": { + hidden: true, + type: "number", + description: "Only cache AttestationData since clockSlot - attDataCacheSlotDistance", + group: "chain", + }, + "chain.computeUnrealized": { hidden: true, type: "boolean", diff --git a/packages/cli/src/options/beaconNodeOptions/network.ts b/packages/cli/src/options/beaconNodeOptions/network.ts index 6e9b54e73c17..7ed4d916194e 100644 --- a/packages/cli/src/options/beaconNodeOptions/network.ts +++ b/packages/cli/src/options/beaconNodeOptions/network.ts @@ -23,6 +23,7 @@ export type NetworkArgs = { "network.gossipsubDHigh": number; "network.gossipsubAwaitHandler": boolean; "network.rateLimitMultiplier": number; + "network.maxGossipTopicConcurrency"?: number; /** @deprecated This option is deprecated and should be removed in next major release. */ "network.requestCountPeerLimit": number; @@ -67,6 +68,7 @@ export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { gossipsubAwaitHandler: args["network.gossipsubAwaitHandler"], mdns: args["mdns"], rateLimitMultiplier: args["network.rateLimitMultiplier"], + maxGossipTopicConcurrency: args["network.maxGossipTopicConcurrency"], }; } @@ -237,4 +239,10 @@ export const options: CliCommandOptions = { defaultDescription: String(defaultOptions.network.rateLimitMultiplier), group: "network", }, + + "network.maxGossipTopicConcurrency": { + type: "number", + hidden: true, + group: "network", + }, }; diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index 9dc9bb4536a4..45a70bf325bf 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -24,6 +24,7 @@ describe("options / beaconNodeOptions", () => { "chain.proposerBoostEnabled": false, "chain.disableImportExecutionFcU": false, "chain.preaggregateSlotDistance": 1, + "chain.attDataCacheSlotDistance": 2, "chain.computeUnrealized": true, suggestedFeeRecipient: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "chain.assertCorrectProgressiveBalances": true, @@ -84,6 +85,7 @@ describe("options / beaconNodeOptions", () => { "network.gossipsubDHigh": 6, "network.gossipsubAwaitHandler": true, "network.rateLimitMultiplier": 1, + "network.maxGossipTopicConcurrency": 64, "sync.isSingleNode": true, "sync.disableProcessAsChainSegment": true, @@ -111,6 +113,7 @@ describe("options / beaconNodeOptions", () => { proposerBoostEnabled: false, disableImportExecutionFcU: false, preaggregateSlotDistance: 1, + attDataCacheSlotDistance: 2, computeUnrealized: true, safeSlotsToImportOptimistically: 256, suggestedFeeRecipient: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", @@ -173,6 +176,7 @@ describe("options / beaconNodeOptions", () => { gossipsubAwaitHandler: true, mdns: false, rateLimitMultiplier: 1, + maxGossipTopicConcurrency: 64, }, sync: { isSingleNode: true, diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index 2e7291c751e2..3419f48e28df 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -478,7 +478,7 @@ export class ForkChoice implements IForkChoice { * The supplied `attestation` **must** pass the `in_valid_indexed_attestation` function as it * will not be run here. */ - onAttestation(attestation: phase0.IndexedAttestation, attDataRoot?: string, forceImport?: boolean): void { + onAttestation(attestation: phase0.IndexedAttestation, attDataRoot: string, forceImport?: boolean): void { // Ignore any attestations to the zero hash. // // This is an edge case that results from the spec aliasing the zero hash to the genesis @@ -913,7 +913,7 @@ export class ForkChoice implements IForkChoice { slot: Slot, blockRootHex: string, targetEpoch: Epoch, - attDataRoot?: string, + attDataRoot: string, // forceImport attestation even if too old, mostly used in spec tests forceImport?: boolean ): void { @@ -931,19 +931,8 @@ export class ForkChoice implements IForkChoice { }); } - const attestationData = indexedAttestation.data; - // AttestationData is expected to internally cache its root to make this hashTreeRoot() call free - const attestationCacheKey = attDataRoot ?? toHexString(ssz.phase0.AttestationData.hashTreeRoot(attestationData)); - - if (!this.validatedAttestationDatas.has(attestationCacheKey)) { - this.validateAttestationData( - indexedAttestation.data, - slot, - blockRootHex, - targetEpoch, - attestationCacheKey, - forceImport - ); + if (!this.validatedAttestationDatas.has(attDataRoot)) { + this.validateAttestationData(indexedAttestation.data, slot, blockRootHex, targetEpoch, attDataRoot, forceImport); } } @@ -952,7 +941,7 @@ export class ForkChoice implements IForkChoice { slot: Slot, beaconBlockRootHex: string, targetEpoch: Epoch, - attestationCacheKey: string, + attDataRoot: string, // forceImport attestation even if too old, mostly used in spec tests forceImport?: boolean ): void { @@ -1054,7 +1043,7 @@ export class ForkChoice implements IForkChoice { }); } - this.validatedAttestationDatas.add(attestationCacheKey); + this.validatedAttestationDatas.add(attDataRoot); } /** diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index a0c6a1465bea..73780687ca72 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -113,7 +113,7 @@ export interface IForkChoice { * The supplied `attestation` **must** pass the `in_valid_indexed_attestation` function as it * will not be run here. */ - onAttestation(attestation: phase0.IndexedAttestation, attDataRoot?: string, forceImport?: boolean): void; + onAttestation(attestation: phase0.IndexedAttestation, attDataRoot: string, forceImport?: boolean): void; /** * Register attester slashing in order not to consider their votes in `getHead` * diff --git a/packages/fork-choice/test/perf/forkChoice/forkChoice.test.ts b/packages/fork-choice/test/perf/forkChoice/forkChoice.test.ts index aa1bb495fcb7..b1300ddf43e1 100644 --- a/packages/fork-choice/test/perf/forkChoice/forkChoice.test.ts +++ b/packages/fork-choice/test/perf/forkChoice/forkChoice.test.ts @@ -3,7 +3,7 @@ import {config} from "@lodestar/config/default"; import {AttestationData, IndexedAttestation} from "@lodestar/types/phase0"; import {ATTESTATION_SUBNET_COUNT} from "@lodestar/params"; import {ssz} from "@lodestar/types"; -import {fromHexString} from "@chainsafe/ssz"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {ExecutionStatus, ForkChoice, IForkChoiceStore, ProtoBlock, ProtoArray} from "../../../src/index.js"; describe("ForkChoice", () => { @@ -160,7 +160,7 @@ describe("ForkChoice", () => { }, fn: (allAttestationsPerSlot) => { for (const attestation of allAttestationsPerSlot) { - forkchoice.onAttestation(attestation); + forkchoice.onAttestation(attestation, toHexString(ssz.phase0.AttestationData.hashTreeRoot(attestation.data))); } }, }); diff --git a/packages/state-transition/src/signatureSets/indexedAttestation.ts b/packages/state-transition/src/signatureSets/indexedAttestation.ts index 1f89fe701176..b5c48a20c9d4 100644 --- a/packages/state-transition/src/signatureSets/indexedAttestation.ts +++ b/packages/state-transition/src/signatureSets/indexedAttestation.ts @@ -1,23 +1,33 @@ import {DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; import {allForks, phase0, ssz} from "@lodestar/types"; -import {computeSigningRoot, computeStartSlotAtEpoch, ISignatureSet, SignatureSetType} from "../util/index.js"; +import { + computeSigningRoot, + computeStartSlotAtEpoch, + createAggregateSignatureSetFromComponents, + ISignatureSet, +} from "../util/index.js"; import {CachedBeaconStateAllForks} from "../types.js"; +export function getAttestationDataSigningRoot( + state: CachedBeaconStateAllForks, + data: phase0.AttestationData +): Uint8Array { + const slot = computeStartSlotAtEpoch(data.target.epoch); + const domain = state.config.getDomain(state.slot, DOMAIN_BEACON_ATTESTER, slot); + + return computeSigningRoot(ssz.phase0.AttestationData, data, domain); +} + export function getAttestationWithIndicesSignatureSet( state: CachedBeaconStateAllForks, attestation: Pick, - indices: number[] + attestingIndices: number[] ): ISignatureSet { - const {epochCtx} = state; - const slot = computeStartSlotAtEpoch(attestation.data.target.epoch); - const domain = state.config.getDomain(state.slot, DOMAIN_BEACON_ATTESTER, slot); - - return { - type: SignatureSetType.aggregate, - pubkeys: indices.map((i) => epochCtx.index2pubkey[i]), - signingRoot: computeSigningRoot(ssz.phase0.AttestationData, attestation.data, domain), - signature: attestation.signature, - }; + return createAggregateSignatureSetFromComponents( + attestingIndices.map((i) => state.epochCtx.index2pubkey[i]), + getAttestationDataSigningRoot(state, attestation.data), + attestation.signature + ); } export function getIndexedAttestationSignatureSet( diff --git a/packages/state-transition/src/util/signatureSets.ts b/packages/state-transition/src/util/signatureSets.ts index 7342efc4b757..770f66e7ecd9 100644 --- a/packages/state-transition/src/util/signatureSets.ts +++ b/packages/state-transition/src/util/signatureSets.ts @@ -36,3 +36,29 @@ export function verifySignatureSet(signatureSet: ISignatureSet): boolean { throw Error("Unknown signature set type"); } } + +export function createSingleSignatureSetFromComponents( + pubkey: PublicKey, + signingRoot: Root, + signature: Uint8Array +): ISignatureSet { + return { + type: SignatureSetType.single, + pubkey, + signingRoot, + signature, + }; +} + +export function createAggregateSignatureSetFromComponents( + pubkeys: PublicKey[], + signingRoot: Root, + signature: Uint8Array +): ISignatureSet { + return { + type: SignatureSetType.aggregate, + pubkeys, + signingRoot, + signature, + }; +} diff --git a/packages/state-transition/test/perf/util/signingRoot.test.ts b/packages/state-transition/test/perf/util/signingRoot.test.ts new file mode 100644 index 000000000000..96684c753364 --- /dev/null +++ b/packages/state-transition/test/perf/util/signingRoot.test.ts @@ -0,0 +1,92 @@ +import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {digest} from "@chainsafe/as-sha256"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {phase0, ssz} from "@lodestar/types"; +import {computeSigningRoot} from "../../../src/util/signingRoot.js"; + +/** + * As of Apr 2023, when we apply new gossip queues we process all gossip attestations and computeSiningRoot may take up to 6% of cpu. + * The below benchmark results show that if we use Buffer.toString(base64) against serialized attestation data, it is still way cheaper + * than computeSigningRoot. + * Based on that we can cache attestation data as string in order to avoid recomputing signing root when validating gossip attestations. + * computeSigningRoot + ✔ computeSigningRoot for AttestationData 94788.17 ops/s 10.54984 us/op - 901 runs 10.0 s + ✔ hash AttestationData serialized data then Buffer.toString(base64 509425.9 ops/s 1.962994 us/op - 4856 runs 10.0 s + ✔ toHexString serialized data 727592.3 ops/s 1.374396 us/op - 6916 runs 10.0 s + ✔ Buffer.toString(base64) 2570800 ops/s 388.9840 ns/op - 24628 runs 10.1 s + */ +describe("computeSigningRoot", function () { + setBenchOpts({ + minMs: 10_000, + }); + + const type = ssz.phase0.AttestationData; + const seedObject: phase0.AttestationData = { + slot: 6118259, + index: 46, + beaconBlockRoot: fromHexString("0x94cef26d543b20568a4bbb77ae2ba203826912065348613a437a9106142aff85"), + source: { + epoch: 191194, + root: fromHexString("0x1a955a91af4ee915c1f267f0026668c58237c1a23bd6c106ef05459741a9171c"), + }, + target: { + epoch: 191195, + root: fromHexString("0x48db1209cd969a1a74eb19d1c5e24021d3a4ac45b8b1b2c1b0e8b0c1b0e8b0c1"), + }, + }; + + const bytes = type.serialize(seedObject); + const domain = new Uint8Array(32); + itBench({ + id: "computeSigningRoot for AttestationData", + fn: () => { + for (let i = 0; i < 1000; i++) { + computeSigningRoot(type, clone(seedObject), domain); + } + }, + runsFactor: 1000, + }); + + itBench({ + id: "hash AttestationData serialized data then Buffer.toString(base64)", + fn: () => { + for (let i = 0; i < 1000; i++) { + clone(seedObject); + Buffer.from(digest(bytes)).toString("base64"); + } + }, + runsFactor: 1000, + }); + + itBench({ + id: "toHexString serialized data", + fn: () => { + for (let i = 0; i < 1000; i++) { + clone(seedObject); + toHexString(bytes); + } + }, + runsFactor: 1000, + }); + + itBench({ + id: "Buffer.toString(base64)", + fn: () => { + for (let i = 0; i < 1000; i++) { + clone(seedObject); + Buffer.from(bytes).toString("base64"); + } + }, + runsFactor: 1000, + }); +}); + +function clone(sszObject: phase0.AttestationData): phase0.AttestationData { + return { + slot: sszObject.slot, + index: sszObject.index, + beaconBlockRoot: sszObject.beaconBlockRoot, + source: sszObject.source, + target: sszObject.target, + }; +} diff --git a/packages/types/src/types.ts b/packages/types/src/types.ts index f6be3fbaa347..44acb7e61455 100644 --- a/packages/types/src/types.ts +++ b/packages/types/src/types.ts @@ -1,3 +1,5 @@ +import {Slot} from "./primitive/types.js"; + export * from "./primitive/types.js"; export {ts as phase0} from "./phase0/index.js"; export {ts as altair} from "./altair/index.js"; @@ -15,3 +17,5 @@ export enum BlockSource { builder = "builder", engine = "engine", } + +export type SlotRootHex = {slot: Slot; root: RootHex};