From b32b6fd4c068d9221b5ab24b6f16dd2af43f4933 Mon Sep 17 00:00:00 2001 From: tersec Date: Tue, 17 Dec 2024 13:09:17 +0000 Subject: [PATCH 1/3] implement EIP-7691 blob sidecar req/resp endpoints --- beacon_chain/spec/datatypes/constants.nim | 7 + beacon_chain/spec/network.nim | 2 +- beacon_chain/spec/presets.nim | 2 + beacon_chain/sync/sync_protocol.nim | 150 ++++++++++++++++++++-- 4 files changed, 148 insertions(+), 13 deletions(-) diff --git a/beacon_chain/spec/datatypes/constants.nim b/beacon_chain/spec/datatypes/constants.nim index cb4c4761a9..4db4ae9d06 100644 --- a/beacon_chain/spec/datatypes/constants.nim +++ b/beacon_chain/spec/datatypes/constants.nim @@ -86,3 +86,10 @@ const # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/electra/beacon-chain.md#withdrawal-prefixes COMPOUNDING_WITHDRAWAL_PREFIX* = 0x02 + + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/electra/beacon-chain.md#execution-1 + MAX_BLOBS_PER_BLOCK_ELECTRA* = 9'u64 + + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/electra/p2p-interface.md#configuration + MAX_REQUEST_BLOB_SIDECARS_ELECTRA* = + MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_ELECTRA diff --git a/beacon_chain/spec/network.nim b/beacon_chain/spec/network.nim index 373f99ca06..3dabaa73af 100644 --- a/beacon_chain/spec/network.nim +++ b/beacon_chain/spec/network.nim @@ -37,7 +37,7 @@ const # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.9/specs/_features/eip7594/p2p-interface.md#configuration MAX_REQUEST_DATA_COLUMN_SIDECARS*: uint64 = MAX_REQUEST_BLOCKS_DENEB * NUMBER_OF_COLUMNS - + defaultEth2TcpPort* = 9000 defaultEth2TcpPortDesc* = $defaultEth2TcpPort diff --git a/beacon_chain/spec/presets.nim b/beacon_chain/spec/presets.nim index 697f20cf46..76b1a9e00b 100644 --- a/beacon_chain/spec/presets.nim +++ b/beacon_chain/spec/presets.nim @@ -824,6 +824,8 @@ proc readRuntimeConfig*( checkCompatibility MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK, "MAX_REQUEST_BLOB_SIDECARS" checkCompatibility BLOB_SIDECAR_SUBNET_COUNT + checkCompatibility MAX_BLOBS_PER_BLOCK_ELECTRA + checkCompatibility MAX_REQUEST_BLOB_SIDECARS_ELECTRA # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/phase0/fork-choice.md#configuration # Isn't being used as a preset in the usual way: at any time, there's one correct value diff --git a/beacon_chain/sync/sync_protocol.nim b/beacon_chain/sync/sync_protocol.nim index 00aa946e72..f544198ac8 100644 --- a/beacon_chain/sync/sync_protocol.nim +++ b/beacon_chain/sync/sync_protocol.nim @@ -91,10 +91,10 @@ proc readChunkPayload*( await conn.readExactly(addr contextBytes, sizeof contextBytes) except CatchableError: return neterr UnexpectedEOF - let contextFork = + let contextFork = peer.network.forkDigests[].consensusForkForDigest(contextBytes).valueOr: return neterr InvalidContextBytes - + withConsensusFork(contextFork): when consensusFork >= ConsensusFork.Fulu: let res = await readChunkPayload(conn, peer, DataColumnSidecar) @@ -259,7 +259,7 @@ p2pProtocol BeaconSync(version = 1, # client call that returns `seq[ref BlobSidecar]` will # will be generated by the libp2p macro - we guarantee that seq items # are `not-nil` in the implementation - trace "got blobs range request", peer, len = blobIds.len + trace "got v1 blobs range request", peer, len = blobIds.len if blobIds.len == 0: raise newException(InvalidInputsError, "No blobs requested") @@ -289,7 +289,7 @@ p2pProtocol BeaconSync(version = 1, peer.network.forkDigestAtEpoch(blockRef.slot.epoch).data) inc found - debug "Blob root request done", + debug "Blob root v1 request done", peer, roots = blobIds.len, count, found # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/p2p-interface.md#blobsidecarsbyrange-v1 @@ -309,7 +309,7 @@ p2pProtocol BeaconSync(version = 1, # will be generated by the libp2p macro - we guarantee that seq items # are `not-nil` in the implementation - trace "got blobs range request", peer, startSlot, count = reqCount + trace "got v1 blobs range request", peer, startSlot, count = reqCount if reqCount == 0: raise newException(InvalidInputsError, "Empty range requested") @@ -361,7 +361,133 @@ p2pProtocol BeaconSync(version = 1, else: break - debug "BlobSidecar range request done", + debug "BlobSidecar v1 range request done", + peer, startSlot, count = reqCount, found + + # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/p2p-interface.md#blobsidecarsbyroot-v1 + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/electra/p2p-interface.md#blobsidecarsbyroot-v2 + proc blobSidecarsByRoot_v2( + peer: Peer, + blobIds: BlobIdentifierList, + response: MultipleChunksResponse[ + ref BlobSidecar, Limit(MAX_REQUEST_BLOB_SIDECARS_ELECTRA)]) + {.async, libp2pProtocol("blob_sidecars_by_root", 2).} = + # TODO Semantically, this request should return a non-ref, but doing so + # runs into extreme inefficiency due to the compiler introducing + # hidden copies - in future nim versions with move support, this should + # be revisited + # TODO This code is more complicated than it needs to be, since the type + # of the multiple chunks response is not actually used in this server + # implementation (it's used to derive the signature of the client + # function, not in the code below!) + # TODO although you can't tell from this function definition, a magic + # client call that returns `seq[ref BlobSidecar]` will + # will be generated by the libp2p macro - we guarantee that seq items + # are `not-nil` in the implementation + trace "got v2 blobs range request", peer, len = blobIds.len + if blobIds.len == 0: + raise newException(InvalidInputsError, "No blobs requested") + + let + dag = peer.networkState.dag + count = blobIds.len + + var + found = 0 + bytes: seq[byte] + + for i in 0..= dag.head.slot.epoch: + GENESIS_EPOCH + else: + dag.head.slot.epoch - dag.cfg.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS + + if startSlot.epoch < epochBoundary: + raise newException(ResourceUnavailableError, BlobsOutOfRange) + + var blockIds: array[int(MAX_REQUEST_BLOB_SIDECARS_ELECTRA), BlockId] + let + count = int min(reqCount, blockIds.lenu64) + endIndex = count - 1 + startIndex = + dag.getBlockRange(startSlot, 1, blockIds.toOpenArray(0, endIndex)) + + var + found = 0 + bytes: seq[byte] + + for i in startIndex..endIndex: + for j in 0..= dag.cfg.BELLATRIX_FORK_EPOCH and + not dag.head.executionValid: + continue + + let uncompressedLen = uncompressedLenFramed(bytes).valueOr: + warn "Cannot read blobs sidecar size, database corrupt?", + bytes = bytes.len(), blck = shortLog(blockIds[i]) + continue + + # TODO extract from libp2pProtocol + peer.awaitQuota(blobResponseCost, "blobs_sidecars_by_range/2") + peer.network.awaitQuota(blobResponseCost, "blobs_sidecars_by_range/2") + + await response.writeBytesSZ( + uncompressedLen, bytes, + peer.network.forkDigestAtEpoch(blockIds[i].slot.epoch).data) + inc found + else: + break + + debug "BlobSidecar v2 range request done", peer, startSlot, count = reqCount, found # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.8/specs/_features/eip7594/p2p-interface.md#datacolumnsidecarsbyroot-v1 @@ -388,7 +514,7 @@ p2pProtocol BeaconSync(version = 1, bytes: seq[byte] for i in 0..= dag.head.slot.epoch: @@ -486,7 +612,7 @@ p2pProtocol BeaconSync(version = 1, debug "Data column range request done", peer, startSlot, count = reqCount, columns = reqColumns, found -proc init*(T: type BeaconSync.NetworkState, dag: ChainDAGRef): T = +func init*(T: type BeaconSync.NetworkState, dag: ChainDAGRef): T = T( dag: dag, ) From 6ce1fb07b4407da442c04c63f5558fbc38dc8a06 Mon Sep 17 00:00:00 2001 From: tersec Date: Wed, 18 Dec 2024 09:40:52 +0000 Subject: [PATCH 2/3] refactor common code out of the blob root/range request handlers --- beacon_chain/sync/sync_protocol.nim | 275 ++++++++++------------------ 1 file changed, 101 insertions(+), 174 deletions(-) diff --git a/beacon_chain/sync/sync_protocol.nim b/beacon_chain/sync/sync_protocol.nim index f544198ac8..2639c5ea30 100644 --- a/beacon_chain/sync/sync_protocol.nim +++ b/beacon_chain/sync/sync_protocol.nim @@ -107,6 +107,99 @@ proc readChunkPayload*( {.pop.} # TODO fix p2p macro for raises +template getBlobSidecarsByRoot( + versionNumber: static string, peer: Peer, dag: ChainDAGRef, response: auto, + blobIds: BlobIdentifierList) = + trace "got v" & versionNumber & " blobs range request", + peer, len = blobIds.len + if blobIds.len == 0: + raise newException(InvalidInputsError, "No blobs requested") + + let count = blobIds.len + + var + found = 0 + bytes: seq[byte] + + for i in 0..= dag.head.slot.epoch: + GENESIS_EPOCH + else: + dag.head.slot.epoch - dag.cfg.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS + + if startSlot.epoch < epochBoundary: + raise newException(ResourceUnavailableError, BlobsOutOfRange) + + var blockIds: array[int(maxReqSidecars), BlockId] + let + count = int min(reqCount, blockIds.lenu64) + endIndex = count - 1 + startIndex = + dag.getBlockRange(startSlot, 1, blockIds.toOpenArray(0, endIndex)) + + var + found = 0 + bytes: seq[byte] + + for i in startIndex..endIndex: + for j in 0..= dag.head.slot.epoch: - GENESIS_EPOCH - else: - dag.head.slot.epoch - dag.cfg.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS - - if startSlot.epoch < epochBoundary: - raise newException(ResourceUnavailableError, BlobsOutOfRange) - - var blockIds: array[int(MAX_REQUEST_BLOB_SIDECARS), BlockId] - let - count = int min(reqCount, blockIds.lenu64) - endIndex = count - 1 - startIndex = - dag.getBlockRange(startSlot, 1, blockIds.toOpenArray(0, endIndex)) - - var - found = 0 - bytes: seq[byte] - - for i in startIndex..endIndex: - for j in 0..= dag.cfg.BELLATRIX_FORK_EPOCH and - not dag.head.executionValid: - continue - - let uncompressedLen = uncompressedLenFramed(bytes).valueOr: - warn "Cannot read blobs sidecar size, database corrupt?", - bytes = bytes.len(), blck = shortLog(blockIds[i]) - continue - - # TODO extract from libp2pProtocol - peer.awaitQuota(blobResponseCost, "blobs_sidecars_by_range/1") - peer.network.awaitQuota(blobResponseCost, "blobs_sidecars_by_range/1") - - await response.writeBytesSZ( - uncompressedLen, bytes, - peer.network.forkDigestAtEpoch(blockIds[i].slot.epoch).data) - inc found - else: - break - - debug "BlobSidecar v1 range request done", - peer, startSlot, count = reqCount, found + await getBlobSidecarsByRange( + "1", peer, peer.networkState.dag, response, startSlot, reqCount, + MAX_BLOBS_PER_BLOCK, MAX_REQUEST_BLOB_SIDECARS) # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/p2p-interface.md#blobsidecarsbyroot-v1 # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/electra/p2p-interface.md#blobsidecarsbyroot-v2 @@ -384,38 +394,7 @@ p2pProtocol BeaconSync(version = 1, # client call that returns `seq[ref BlobSidecar]` will # will be generated by the libp2p macro - we guarantee that seq items # are `not-nil` in the implementation - trace "got v2 blobs range request", peer, len = blobIds.len - if blobIds.len == 0: - raise newException(InvalidInputsError, "No blobs requested") - - let - dag = peer.networkState.dag - count = blobIds.len - - var - found = 0 - bytes: seq[byte] - - for i in 0..= dag.head.slot.epoch: - GENESIS_EPOCH - else: - dag.head.slot.epoch - dag.cfg.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS - - if startSlot.epoch < epochBoundary: - raise newException(ResourceUnavailableError, BlobsOutOfRange) - - var blockIds: array[int(MAX_REQUEST_BLOB_SIDECARS_ELECTRA), BlockId] - let - count = int min(reqCount, blockIds.lenu64) - endIndex = count - 1 - startIndex = - dag.getBlockRange(startSlot, 1, blockIds.toOpenArray(0, endIndex)) - - var - found = 0 - bytes: seq[byte] - - for i in startIndex..endIndex: - for j in 0..= dag.cfg.BELLATRIX_FORK_EPOCH and - not dag.head.executionValid: - continue - - let uncompressedLen = uncompressedLenFramed(bytes).valueOr: - warn "Cannot read blobs sidecar size, database corrupt?", - bytes = bytes.len(), blck = shortLog(blockIds[i]) - continue - - # TODO extract from libp2pProtocol - peer.awaitQuota(blobResponseCost, "blobs_sidecars_by_range/2") - peer.network.awaitQuota(blobResponseCost, "blobs_sidecars_by_range/2") - - await response.writeBytesSZ( - uncompressedLen, bytes, - peer.network.forkDigestAtEpoch(blockIds[i].slot.epoch).data) - inc found - else: - break - - debug "BlobSidecar v2 range request done", - peer, startSlot, count = reqCount, found + await getBlobSidecarsByRange( + "2", peer, peer.networkState.dag, response, startSlot, reqCount, + MAX_BLOBS_PER_BLOCK_ELECTRA, MAX_REQUEST_BLOB_SIDECARS_ELECTRA) # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.8/specs/_features/eip7594/p2p-interface.md#datacolumnsidecarsbyroot-v1 proc dataColumnSidecarsByRoot( From dc385badfe17fb3e50f3a15122b78848fea7252d Mon Sep 17 00:00:00 2001 From: tersec Date: Wed, 18 Dec 2024 11:59:25 +0000 Subject: [PATCH 3/3] use template instead of proc --- beacon_chain/sync/sync_protocol.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beacon_chain/sync/sync_protocol.nim b/beacon_chain/sync/sync_protocol.nim index 2639c5ea30..43d20fe0a7 100644 --- a/beacon_chain/sync/sync_protocol.nim +++ b/beacon_chain/sync/sync_protocol.nim @@ -144,10 +144,10 @@ template getBlobSidecarsByRoot( debug "Blob root v" & versionNumber & " request done", peer, roots = blobIds.len, count, found -proc getBlobSidecarsByRange( +template getBlobSidecarsByRange( versionNumber: static string, peer: Peer, dag: ChainDAGRef, response: auto, startSlot: Slot, reqCount: uint64, blobsPerBlock: static uint64, - maxReqSidecars: static uint64) {.async.} = + maxReqSidecars: static uint64) = trace "got v" & versionNumber & " blobs range request", peer, startSlot, count = reqCount if reqCount == 0: @@ -370,7 +370,7 @@ p2pProtocol BeaconSync(version = 1, # client call that returns `seq[ref BlobSidecar]` will # will be generated by the libp2p macro - we guarantee that seq items # are `not-nil` in the implementation - await getBlobSidecarsByRange( + getBlobSidecarsByRange( "1", peer, peer.networkState.dag, response, startSlot, reqCount, MAX_BLOBS_PER_BLOCK, MAX_REQUEST_BLOB_SIDECARS) @@ -413,7 +413,7 @@ p2pProtocol BeaconSync(version = 1, # client call that returns `seq[ref BlobSidecar]` will # will be generated by the libp2p macro - we guarantee that seq items # are `not-nil` in the implementation - await getBlobSidecarsByRange( + getBlobSidecarsByRange( "2", peer, peer.networkState.dag, response, startSlot, reqCount, MAX_BLOBS_PER_BLOCK_ELECTRA, MAX_REQUEST_BLOB_SIDECARS_ELECTRA)