diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index a48cd64b88..dd1b338dd6 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -107,6 +107,16 @@ AllTests-mainnet + Electra toSignedBlindedBeaconBlock OK + Fulu toSignedBlindedBeaconBlock OK ``` +## BlobQuarantine data structure test suite [Preset: mainnet] +```diff ++ overfill protection test OK ++ popSidecars()/hasSidecars() return []/true on block without blobs OK ++ pruneAfterFinalization() test OK ++ put() duplicate items should not affect counters OK ++ put()/fetchMissingSidecars/remove test OK ++ put()/hasSidecar(index, slot, proposer_index)/remove() test OK ++ put(sidecar)/put([sidecars])/hasSidecars/popSidecars/remove() test OK +``` ## Block pool altair processing [Preset: mainnet] ```diff + Invalid signatures [Preset: mainnet] OK @@ -146,6 +156,19 @@ AllTests-mainnet + atSlot sanity OK + parent sanity OK ``` +## ColumnQuarantine data structure test suite [Preset: mainnet] +```diff ++ ColumnMap test OK ++ overfill protection test OK ++ popSidecars()/hasSidecars() return []/true on block without columns OK ++ pruneAfterFinalization() test OK ++ put() duplicate items should not affect counters OK ++ put()/fetchMissingSidecars/remove test [node] OK ++ put()/fetchMissingSidecars/remove test [supernode] OK ++ put()/hasSidecar(index, slot, proposer_index)/remove() test OK ++ put(sidecar)/put([sidecars])/hasSidecars/popSidecars/remove() [node] test OK ++ put(sidecar)/put([sidecars])/hasSidecars/popSidecars/remove() [supernode] test OK +``` ## Combined scenarios [Beacon Node] [Preset: mainnet] ```diff + ImportKeystores should not be blocked by fee recipient setting [Beacon Node] [Preset: main OK diff --git a/beacon_chain/consensus_object_pools/blob_quarantine.nim b/beacon_chain/consensus_object_pools/blob_quarantine.nim index 12da2ca4ff..6b36d0741b 100644 --- a/beacon_chain/consensus_object_pools/blob_quarantine.nim +++ b/beacon_chain/consensus_object_pools/blob_quarantine.nim @@ -8,39 +8,179 @@ {.push raises: [].} import - std/tables, - ../spec/helpers + stew/bitops2, + std/[sets, tables], + results, + ../spec/datatypes/[deneb, electra, fulu], + ../spec/[presets, helpers] -from std/sequtils import mapIt +from std/sequtils import mapIt, toSeq from std/strutils import join -func maxBlobs(MAX_BLOBS_PER_BLOCK_ELECTRA: uint64): uint64 = - # Same limit as `MaxOrphans` in `block_quarantine`; - # blobs may arrive before an orphan is tagged `blobless` - 3 * SLOTS_PER_EPOCH * MAX_BLOBS_PER_BLOCK_ELECTRA +export results + +static: + doAssert(NUMBER_OF_COLUMNS == 2 * 64, "ColumnMap should be updated") type - BlobQuarantine* = object - maxBlobs: uint64 - blobs*: - OrderedTable[(Eth2Digest, BlobIndex, KzgCommitment), ref BlobSidecar] - onBlobSidecarCallback*: OnBlobSidecarCallback + ColumnMap* = object + data: array[2, uint64] + + RootTableRecord[A] = object + sidecars: seq[ref A] + count: int + + SidecarQuarantine[A, B] = object + maxSidecarsCount: int + maxSidecarsPerBlockCount: int + sidecarsCount: int + custodyColumns: seq[ColumnIndex] + custodyMap: ColumnMap + roots: Table[Eth2Digest, RootTableRecord[A]] + usage: OrderedSet[Eth2Digest] + indexMap: seq[int] + onSidecarCallback*: B + + OnBlobSidecarCallback* = proc( + data: BlobSidecarInfoObject) {.gcsafe, raises: [].} + OnDataColumnSidecarCallback* = proc( + data: DataColumnSidecar) {.gcsafe, raises: [].} - BlobFetchRecord* = object - block_root*: Eth2Digest - indices*: seq[BlobIndex] + BlobQuarantine* = + SidecarQuarantine[BlobSidecar, OnBlobSidecarCallback] + ColumnQuarantine* = + SidecarQuarantine[DataColumnSidecar, OnDataColumnSidecarCallback] - OnBlobSidecarCallback = proc( - data: BlobSidecarInfoObject) {.gcsafe, raises: [].} +func init*(t: typedesc[ColumnMap], columns: openArray[ColumnIndex]): ColumnMap = + var res: ColumnMap + for column in columns: + let + index = int(uint64(column) shr 6) + offset = int(uint64(column) and 0x3F'u64) + res.data[index].setBit(offset) + res + +func `and`*(a, b: ColumnMap): ColumnMap = + ColumnMap(data: [a.data[0] and b.data[0], a.data[1] and b.data[1]]) + +iterator items*(a: ColumnMap): ColumnIndex = + var + data0 = a.data[0] + data1 = a.data[1] + + while data0 != 0'u64: + let + # t = data0 and -data0 + t = data0 and ((0xFFFF_FFFF_FFFF_FFFF'u64 - data0) + 1'u64) + res = firstOne(data0) + yield ColumnIndex(res - 1) + data0 = data0 xor t + + while data1 != 0'u64: + let + # t = data0 and -data0 + t = data1 and ((0xFFFF_FFFF_FFFF_FFFF'u64 - data1) + 1'u64) + res = firstOne(data1) + yield ColumnIndex(64 + res - 1) + data1 = data1 xor t + +func `$`*(a: ColumnMap): string = + "[" & a.items().toSeq().mapIt($it).join(", ") & "]" + +func maxSidecars(maxSidecarsPerBlock: uint64): int = + # Same limit as `MaxOrphans` in `block_quarantine`; + # blobs may arrive before an orphan is tagged `blobless` + 3 * int(SLOTS_PER_EPOCH) * int(maxSidecarsPerBlock) func shortLog*(x: seq[BlobIndex]): string = "<" & x.mapIt($it).join(", ") & ">" -func shortLog*(x: seq[BlobFetchRecord]): string = - "[" & x.mapIt(shortLog(it.block_root) & shortLog(it.indices)).join(", ") & "]" +func init[A, B]( + t: typedesc[RootTableRecord], + q: SidecarQuarantine[A, B] +): RootTableRecord[A] = + RootTableRecord[A]( + sidecars: newSeq[ref A](q.maxSidecarsPerBlockCount), count: 0) + +func len*[A, B](quarantine: SidecarQuarantine[A, B]): int = + quarantine.sidecarsCount + +func `$`*[A](r: RootTableRecord[A]): string = + if len(r.sidecars) == 0: + return "" + r.sidecars.mapIt(if isNil(it): "." else: "x").join("") + +func removeRoot[A, B]( + quarantine: var SidecarQuarantine[A, B], + blockRoot: Eth2Digest +) = + var + rootRecord: RootTableRecord[A] + + if quarantine.roots.pop(blockRoot, rootRecord): + for index in 0 ..< len(rootRecord.sidecars): + if not(rootRecord.sidecars[index].isNil()): + rootRecord.sidecars[index] = nil + dec(quarantine.sidecarsCount) + + quarantine.usage.excl(blockRoot) + +func remove*[A, B]( + quarantine: var SidecarQuarantine[A, B], + blockRoot: Eth2Digest +) = + ## Remove all the data columns or blobs related to the block root ``blockRoot` + ## from the quarantine ``quarantine``. + ## + ## Function do nothing, if ``blockRoot` is not part of the quarantine. + quarantine.removeRoot(blockRoot) + +func pruneRoot[A, B](quarantine: var SidecarQuarantine[A, B]) = + # Remove the all the blobs related to the oldest block root from the + # quarantine ``quarantine``. + if len(quarantine.usage) == 0: + return + var oldestRoot: Eth2Digest + for blockRoot in quarantine.usage: + oldestRoot = blockRoot + break + quarantine.remove(oldestRoot) + +func getIndex(quarantine: BlobQuarantine, index: BlobIndex): int = + quarantine.indexMap[int(index)] + +func getIndex(quarantine: ColumnQuarantine, index: ColumnIndex): int = + quarantine.indexMap[int(index)] -func put*(quarantine: var BlobQuarantine, blobSidecar: ref BlobSidecar) = - if quarantine.blobs.lenu64 >= quarantine.maxBlobs: +template slot(b: BlobSidecar|DataColumnSidecar): Slot = + b.signed_block_header.message.slot + +template proposer_index(b: BlobSidecar|DataColumnSidecar): uint64 = + b.signed_block_header.message.proposer_index + +func put[A, B](record: var RootTableRecord[A], q: var SidecarQuarantine[A, B], + sidecars: openArray[ref A]) = + for sidecar in sidecars: + # Sidecar should pass validation before being added to quarantine, + # so we assume that + # 1. sidecar.index is < MAX_BLOBS_PER_BLOCK for `deneb` and. + # 2. sidecar.index is < MAX_BLOBS_PER_BLOCK_ELECTRA for `electra`. + # 3. sidecar.index is in custody columns set for `fulu`. + let index = q.getIndex(sidecar.index) + doAssert(index >= 0, "Incorrect sidecar index [" & $sidecar.index & "]") + if isNil(record.sidecars[index]): + inc(q.sidecarsCount) + inc(record.count) + record.sidecars[index] = sidecar + +func put*[A, B]( + quarantine: var SidecarQuarantine[A, B], + blockRoot: Eth2Digest, + sidecar: ref A +) = + ## Function adds blob or data column sidecar associated with block root + ## ``blockRoot`` to the quarantine ``quarantine``. + while quarantine.sidecarsCount >= quarantine.maxSidecarsCount: # FIFO if full. For example, sync manager and request manager can race to # put blobs in at the same time, so one gets blob insert -> block resolve # -> blob insert sequence, which leaves garbage blobs. @@ -49,66 +189,410 @@ func put*(quarantine: var BlobQuarantine, blobSidecar: ref BlobSidecar) = # blobs which are correctly signed, point to either correct block roots or a # block root which isn't ever seen, and then are for any reason simply never # used. - var oldest_blob_key: (Eth2Digest, BlobIndex, KzgCommitment) - for k in quarantine.blobs.keys: - oldest_blob_key = k - break - quarantine.blobs.del oldest_blob_key - let block_root = hash_tree_root(blobSidecar.signed_block_header.message) - discard quarantine.blobs.hasKeyOrPut( - (block_root, blobSidecar.index, blobSidecar.kzg_commitment), blobSidecar) - -func hasBlob*( + quarantine.pruneRoot() + + let rootRecord = RootTableRecord.init(quarantine) + quarantine.roots.mgetOrPut(blockRoot, rootRecord).put( + quarantine, [sidecar]) + quarantine.usage.incl(blockRoot) + +func put*[A, B]( + quarantine: var SidecarQuarantine[A, B], + blockRoot: Eth2Digest, + sidecars: openArray[ref A] +) = + ## Function adds number of blobs or data columns sidecars associated to single + ## block with root ``blockRoot`` to the quarantine ``quarantine``. + if len(sidecars) == 0: + return + + while quarantine.sidecarsCount + len(sidecars) >= quarantine.maxSidecarsCount: + # FIFO if full. For example, sync manager and request manager can race to + # put blobs in at the same time, so one gets blob insert -> block resolve + # -> blob insert sequence, which leaves garbage blobs. + # + # This also therefore automatically garbage-collects otherwise valid garbage + # blobs which are correctly signed, point to either correct block roots or a + # block root which isn't ever seen, and then are for any reason simply never + # used. + quarantine.pruneRoot() + + let rootRecord = RootTableRecord.init(quarantine) + + quarantine.roots.mgetOrPut(blockRoot, rootRecord).put( + quarantine, sidecars) + quarantine.usage.incl(blockRoot) + +template hasSidecarImpl( + blockRoot: Eth2Digest, + slot: Slot, + proposerIndex: uint64, + sidecarIndex: typed +): bool = + let rootRecord = quarantine.roots.getOrDefault(blockRoot) + if rootRecord.count == 0: + return false + let index = quarantine.getIndex(index) + if (index == -1) or (isNil(rootRecord.sidecars[index])): + return false + if (rootRecord.sidecars[index][].proposer_index() != proposer_index) or + (rootRecord.sidecars[index][].slot() != slot): + return false + true + +func hasSidecar*( quarantine: BlobQuarantine, + blockRoot: Eth2Digest, slot: Slot, proposer_index: uint64, index: BlobIndex, - kzg_commitment: KzgCommitment): bool = - for blob_sidecar in quarantine.blobs.values: - template block_header: untyped = blob_sidecar.signed_block_header.message - if block_header.slot == slot and - block_header.proposer_index == proposer_index and - blob_sidecar.index == index and - blob_sidecar.kzg_commitment == kzg_commitment: - return true - false - -func popBlobs*( - quarantine: var BlobQuarantine, digest: Eth2Digest, - blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock | - fulu.SignedBeaconBlock): - seq[ref BlobSidecar] = - var r: seq[ref BlobSidecar] = @[] - for idx, kzg_commitment in blck.message.body.blob_kzg_commitments: - var b: ref BlobSidecar - if quarantine.blobs.pop((digest, BlobIndex idx, kzg_commitment), b): - r.add(b) - r - -func hasBlobs*(quarantine: BlobQuarantine, +): bool = + ## Function returns ``true``if quarantine has blob corresponding to specific + ## ``block root``, ``index``, ``slot`` and ``proposer_index``. + hasSidecarImpl(blockRoot, slot, proposer_index, index) + +func hasSidecar*( + quarantine: ColumnQuarantine, + blockRoot: Eth2Digest, + slot: Slot, + proposer_index: uint64, + index: ColumnIndex +): bool = + ## Function returns ``true``if quarantine has column corresponding to specific + ## ``index``, ``slot`` and ``proposer_index``. + hasSidecarImpl(blockRoot, slot, proposer_index, index) + +func hasSidecars*( + quarantine: BlobQuarantine, + blockRoot: Eth2Digest, blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock | - fulu.SignedBeaconBlock): bool = - # Having a fulu SignedBeaconBlock is incorrect atm, but - # shall be fixed once data columns are rebased to fulu - for idx, kzg_commitment in blck.message.body.blob_kzg_commitments: - if (blck.root, BlobIndex idx, kzg_commitment) notin quarantine.blobs: - return false + fulu.SignedBeaconBlock +): bool = + ## Function returns ``true`` if quarantine has all the blobs for block + ## ``blck`` with block root ``blockRoot``. + if len(blck.message.body.blob_kzg_commitments) == 0: + return true + + let record = quarantine.roots.getOrDefault(blockRoot) + if len(record.sidecars) == 0: + # block root not found, record.sidecars sequence was not initialized. + return false + + if record.count < len(blck.message.body.blob_kzg_commitments): + # Quarantine does not hold enough blob sidecars. + return false + true + +func hasSidecars*( + quarantine: ColumnQuarantine, + blockRoot: Eth2Digest, + blck: fulu.SignedBeaconBlock +): bool = + ## Function returns ``true`` if quarantine has all the columns for block + ## ``blck`` with block root ``blockRoot``. + if len(blck.message.body.blob_kzg_commitments) == 0: + return true + + let record = quarantine.roots.getOrDefault(blockRoot) + if len(record.sidecars) == 0: + # block root not found, record.sidecars sequence was not initialized. + return false + + let + supernode = (len(quarantine.custodyColumns) == NUMBER_OF_COLUMNS) + columnsCount = + if supernode: + (NUMBER_OF_COLUMNS div 2 + 1) + else: + len(quarantine.custodyColumns) + + if record.count < columnsCount: + # Quarantine does not hold enough column sidecars. + return false true -func blobFetchRecord*(quarantine: BlobQuarantine, +func hasSidecars*( + quarantine: BlobQuarantine, blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock | - fulu.SignedBeaconBlock): BlobFetchRecord = - var indices: seq[BlobIndex] - for i in 0..= (NUMBER_OF_COLUMNS div 2 + 1), + "Incorrect amount of sidecars in record") + Opt.some(sidecars) + else: + for cindex in quarantine.custodyColumns: + let index = quarantine.getIndex(cindex) + doAssert(not(isNil(record.sidecars[index])), + "Record should not store nil values when record's count is correct") + sidecars.add(record.sidecars[index]) + Opt.some(sidecars) + +func popSidecars*( + quarantine: var BlobQuarantine, + blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock | + fulu.SignedBeaconBlock +): Opt[seq[ref BlobSidecar]] = + ## Alias for `popSidecars()`. + popSidecars(quarantine, blck.root, blck) + +func popSidecars*( + quarantine: var ColumnQuarantine, + blck: fulu.SignedBeaconBlock +): Opt[seq[ref DataColumnSidecar]] = + ## Alias for `popSidecars()`. + popSidecars(quarantine, blck.root, blck) + +func fetchMissingSidecars*( + quarantine: BlobQuarantine, + blockRoot: Eth2Digest, + blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock | + fulu.SignedBeaconBlock +): seq[BlobIdentifier] = + ## Function returns sequence of BlobIdentifiers for blobs which are missing + ## for block root ``blockRoot`` and block ``blck``. + var res: seq[BlobIdentifier] + let record = quarantine.roots.getOrDefault(blockRoot) + + let commitmentsCount = len(blck.message.body.blob_kzg_commitments) + if (commitmentsCount == 0) or (record.count == commitmentsCount): + # Fast-path if ``blck`` does not have any blobs or if quarantine's record + # holds enough blobs. + return res + + for bindex in 0 ..< commitmentsCount: + let index = quarantine.getIndex(BlobIndex(bindex)) + if len(record.sidecars) == 0 or (record.sidecars[index].isNil()): + res.add(BlobIdentifier(block_root: blockRoot, index: BlobIndex(bindex))) + res + +func fetchMissingSidecars*( + quarantine: ColumnQuarantine, + blockRoot: Eth2Digest, + blck: fulu.SignedBeaconBlock, + peerCustodyColumns: openArray[ColumnIndex] = [] +): seq[DataColumnIdentifier] = + ## Function returns sequence of DataColumnIdentifier's for data columns which + ## are missing for block associated with root ``blockRoot`` and block ``blck``. + var res: seq[DataColumnIdentifier] + let record = quarantine.roots.getOrDefault(blockRoot) + + if len(blck.message.body.blob_kzg_commitments) == 0: + # Fast-path if block do not have any columns + return res + + let + supernode = (len(quarantine.custodyColumns) == NUMBER_OF_COLUMNS) + columnsCount = + if supernode: + (NUMBER_OF_COLUMNS div 2 + 1) + else: + len(quarantine.custodyColumns) + + if supernode: + let + columns = + if len(peerCustodyColumns) > 0: + @peerCustodyColumns + else: + quarantine.custodyColumns + if len(record.sidecars) == 0: + var columnsRequested = 0 + for column in columns: + if columnsRequested >= columnsCount: + # We don't need to request more than (NUMBER_OF_COLUMNS div 2 + 1) + # columns. + break + res.add(DataColumnIdentifier(block_root: blockRoot, index: column)) + inc(columnsRequested) + else: + if record.count >= columnsCount: + return res + var columnsRequested = 0 + for column in columns: + if record.count + columnsRequested >= columnsCount: + # We don't need to request more than (NUMBER_OF_COLUMNS div 2 + 1) + # columns. + break + let index = quarantine.getIndex(column) + if (index == -1) or record.sidecars[index].isNil(): + res.add(DataColumnIdentifier(block_root: blockRoot, index: column)) + inc(columnsRequested) + else: + let peerMap = + if len(peerCustodyColumns) > 0: + ColumnMap.init(peerCustodyColumns) + else: + ColumnMap.init(quarantine.custodyColumns) + if len(record.sidecars) == 0: + for column in (peerMap and quarantine.custodyMap).items(): + res.add(DataColumnIdentifier(block_root: blockRoot, index: column)) + else: + for column in (peerMap and quarantine.custodyMap).items(): + let index = quarantine.getIndex(column) + if (index == -1) or (record.sidecars[index].isNil()): + res.add(DataColumnIdentifier(block_root: blockRoot, index: column)) + res + +func pruneAfterFinalization*[A, B]( + quarantine: var SidecarQuarantine[A, B], + epoch: Epoch +) = + let epochSlot = epoch.start_slot() + var + sidecarsCount = 0 + rootsToRemove: seq[Eth2Digest] + + for mkey, mrecord in quarantine.roots.mpairs(): + var removeRoot = false + for index in 0 ..< len(mrecord.sidecars): + if not(isNil(mrecord.sidecars[index])) and + mrecord.sidecars[index][].slot < epochSlot: + removeRoot = true + # Preemptively freeing `ref` object reference. + mrecord.sidecars[index] = nil + inc(sidecarsCount) + if removeRoot: + rootsToRemove.add(mkey) + + for root in rootsToRemove: + quarantine.roots.del(root) + + dec(quarantine.sidecarsCount, sidecarsCount) + +template onBlobSidecarCallback*( + quarantine: BlobQuarantine +): OnBlobSidecarCallback = + quarantine.onSidecarCallback + +template onDataColumnSidecarCallback*( + quarantine: ColumnQuarantine +): OnDataColumnSidecarCallback = + quarantine.onSidecarCallback func init*( - T: type BlobQuarantine, + T: typedesc[BlobQuarantine], cfg: RuntimeConfig, - onBlobSidecarCallback: OnBlobSidecarCallback): T = - T(maxBlobs: cfg.MAX_BLOBS_PER_BLOCK_ELECTRA.maxBlobs(), - onBlobSidecarCallback: onBlobSidecarCallback) + onBlobSidecarCallback: OnBlobSidecarCallback +): BlobQuarantine = + # BlobSidecars maps are trivial, but still useful + var indexMap = newSeqUninit[int](cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) + for index in 0 ..< len(indexMap): + indexMap[index] = index + + let size = maxSidecars(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) + BlobQuarantine( + maxSidecarsPerBlockCount: int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA), + maxSidecarsCount: size, + sidecarsCount: 0, + indexMap: indexMap, + onSidecarCallback: onBlobSidecarCallback + ) + +func init*( + T: typedesc[ColumnQuarantine], + cfg: RuntimeConfig, + custodyColumns: openArray[ColumnIndex], + onBlobSidecarCallback: OnDataColumnSidecarCallback +): ColumnQuarantine = + doAssert(len(custodyColumns) <= NUMBER_OF_COLUMNS) + let size = maxSidecars(NUMBER_OF_COLUMNS) + var indexMap = newSeqUninit[int](NUMBER_OF_COLUMNS) + if len(custodyColumns) < NUMBER_OF_COLUMNS: + for i in 0 ..< len(indexMap): + indexMap[i] = -1 + for index, item in custodyColumns.pairs(): + doAssert(item < uint64(NUMBER_OF_COLUMNS)) + indexMap[int(item)] = index + + ColumnQuarantine( + maxSidecarsPerBlockCount: len(custodyColumns), + maxSidecarsCount: size, + sidecarsCount: 0, + indexMap: indexMap, + custodyColumns: @custodyColumns, + custodyMap: ColumnMap.init(custodyColumns), + onSidecarCallback: onBlobSidecarCallback + ) diff --git a/beacon_chain/gossip_processing/block_processor.nim b/beacon_chain/gossip_processing/block_processor.nim index 6170f5ba98..318cfc837b 100644 --- a/beacon_chain/gossip_processing/block_processor.nim +++ b/beacon_chain/gossip_processing/block_processor.nim @@ -28,7 +28,7 @@ from ../consensus_object_pools/block_pools_types import from ../consensus_object_pools/block_quarantine import addBlobless, addOrphan, addUnviable, pop, removeOrphan from ../consensus_object_pools/blob_quarantine import - BlobQuarantine, hasBlobs, popBlobs, put + BlobQuarantine, popSidecars, put from ../validators/validator_monitor import MsgSource, ValidatorMonitor, registerAttestationInBlock, registerBeaconBlock, registerSyncAggregateInBlock @@ -500,8 +500,7 @@ proc storeBlock( err = r.error() else: if blobsOpt.isSome: - for blobSidecar in blobsOpt.get: - self.blobQuarantine[].put(blobSidecar) + self.blobQuarantine[].put(signedBlock.root, blobsOpt.get) debug "Block quarantined", blockRoot = shortLog(signedBlock.root), blck = shortLog(signedBlock.message), @@ -852,10 +851,10 @@ proc storeBlock( blck = shortLog(forkyBlck), error = res.error() continue - if self.blobQuarantine[].hasBlobs(forkyBlck): - let blobs = self.blobQuarantine[].popBlobs( - forkyBlck.root, forkyBlck) - self[].enqueueBlock(MsgSource.gossip, quarantined, Opt.some(blobs)) + let bres = + self.blobQuarantine[].popSidecars(forkyBlck.root, forkyBlck) + if bres.isSome(): + self[].enqueueBlock(MsgSource.gossip, quarantined, bres) else: discard self.consensusManager.quarantine[].addBlobless( dag.finalizedHead.slot, forkyBlck) diff --git a/beacon_chain/gossip_processing/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim index 3d1382c4de..8ec01b089e 100644 --- a/beacon_chain/gossip_processing/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -241,8 +241,10 @@ proc processSignedBeaconBlock*( let blobs = when typeof(signedBlock).kind >= ConsensusFork.Deneb: - if self.blobQuarantine[].hasBlobs(signedBlock): - Opt.some(self.blobQuarantine[].popBlobs(signedBlock.root, signedBlock)) + let bres = + self.blobQuarantine[].popSidecars(signedBlock.root, signedBlock) + if bres.isSome(): + bres else: discard self.quarantine[].addBlobless(self.dag.finalizedHead.slot, signedBlock) @@ -295,18 +297,17 @@ proc processBlobSidecar*( blob_sidecars_dropped.inc(1, [$v.error[0]]) return v + let block_root = hash_tree_root(block_header) debug "Blob validated, putting in blob quarantine" - self.blobQuarantine[].put(newClone(blobSidecar)) + self.blobQuarantine[].put(block_root, newClone(blobSidecar)) - let block_root = hash_tree_root(block_header) if (let o = self.quarantine[].popBlobless(block_root); o.isSome): let blobless = o.unsafeGet() withBlck(blobless): when consensusFork >= ConsensusFork.Deneb: - if self.blobQuarantine[].hasBlobs(forkyBlck): - self.blockProcessor[].enqueueBlock( - MsgSource.gossip, blobless, - Opt.some(self.blobQuarantine[].popBlobs(block_root, forkyBlck))) + let bres = self.blobQuarantine[].popSidecars(block_root, forkyBlck) + if bres.isSome(): + self.blockProcessor[].enqueueBlock(MsgSource.gossip, blobless, bres) else: discard self.quarantine[].addBlobless( self.dag.finalizedHead.slot, forkyBlck) diff --git a/beacon_chain/gossip_processing/gossip_validation.nim b/beacon_chain/gossip_processing/gossip_validation.nim index 2808e9dab2..255c2f6428 100644 --- a/beacon_chain/gossip_processing/gossip_validation.nim +++ b/beacon_chain/gossip_processing/gossip_validation.nim @@ -424,7 +424,7 @@ proc validateBlobSidecar*( if dag.getBlockRef(block_root).isSome(): return errIgnore("BlobSidecar: already have block") - # This adds KZG commitment matching to the spec gossip validation. It's an + # This adds block root matching to the spec gossip validation. It's an # IGNORE condition, so it shouldn't affect Nimbus's scoring, and when some # (slashable) double proposals happen with blobs present, without this one # or the other block, or potentially both, won't get its full set of blobs @@ -434,9 +434,51 @@ proc validateBlobSidecar*( # # It would be good to fix this more properly, but this has come up often on # Pectra devnet-6. - if blobQuarantine[].hasBlob( - block_header.slot, block_header.proposer_index, blob_sidecar.index, - blob_sidecar.kzg_commitment): + # + # Detailed explanation: + # + # There were regular double-proposer, slashable events (some of which got + # slashed, but that takes at least a couple of slots typically to be noticed, + # it's not instant). What would happen is, Nimbus would be going fine, + # following the chain, until one of these double proposals came up. + # Each had, independently, some set of blobs: + # + # * separately valid block 1, with a set of valid blobs; and + # * separately valid block 2, with a set of valid blobs (different than the + # first set, created by a different node). + # + # Both of these proposals shared a slot and proposer index, because they were + # the same proposer. Indeed, the signatures were all valid too, because, well, + # they were both legitimately running that private key. + # + # But what would happen is, + # * if block 1's blobs came in, and block 1 came in, and block 1 turned out + # to be the one the chain followed, then, great, the IGNORE condition here + # worked fine (WLOG extend to block 2); but + # * if the blobs came in interleaved, this wasn't always true, and, + # crucially, this gossip condition as spec-written prevented Nimbus's + # gossip from being able to collect all the blobs from block 1. + # + # Maybe other clients did/do this by having a very efficient + # request manager-equivalent, I'm not sure. But without something, either + # receiving via gossip or req/resp, Nimbus just got stuck until a suitable + # reorg happened, typically dozens of slots later, because this gossip + # condition prevented it from seeing all the blobs corresponding to either + # block. + # + # Also, it would be basically random chance which, if asked by req/resp, + # of the two different (or more, but the devnet-6 case was two slashable + # blocks at a time) sets of blobs would be returned, so it seemed to + # sometimes have to retry this. All of this took enough time Nimbus lost the + # chain basically deterministically every time this slashable double-proposal + # situation came up. + # + # I don't see anything obviously corresponding to this in the tests, either, + # to show this is otherwise addressed. + + if blobQuarantine[].hasSidecar(block_root, block_header.slot, + block_header.proposer_index, + blob_sidecar.index): return errIgnore("BlobSidecar: already have valid blob from same proposer") # [REJECT] The sidecar's inclusion proof is valid as verified by @@ -522,9 +564,10 @@ proc validateBlobSidecar*( return dag.checkedReject("BlobSidecar: blob invalid") # Send notification about new blob sidecar via callback - if not(isNil(blobQuarantine.onBlobSidecarCallback)): - blobQuarantine.onBlobSidecarCallback BlobSidecarInfoObject( - block_root: hash_tree_root(blob_sidecar.signed_block_header.message), + let onBlobSidecarCallback = blobQuarantine[].onBlobSidecarCallback() + if not(isNil(onBlobSidecarCallback)): + onBlobSidecarCallback BlobSidecarInfoObject( + block_root: block_root, index: blob_sidecar.index, slot: blob_sidecar.signed_block_header.message.slot, kzg_commitment: blob_sidecar.kzg_commitment, diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 02ddec4f1b..66db39eb82 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -444,18 +444,17 @@ proc initFullNode( Future[Result[void, VerifierError]] {.async: (raises: [CancelledError]).} = withBlck(signedBlock): when consensusFork >= ConsensusFork.Deneb: - if not blobQuarantine[].hasBlobs(forkyBlck): + let bres = blobQuarantine[].popSidecars(forkyBlck.root, forkyBlck) + if bres.isSome(): + await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, bres, + maybeFinalized = maybeFinalized) + else: # We don't have all the blobs for this block, so we have # to put it in blobless quarantine. if not quarantine[].addBlobless(dag.finalizedHead.slot, forkyBlck): err(VerifierError.UnviableFork) else: err(VerifierError.MissingParent) - else: - let blobs = blobQuarantine[].popBlobs(forkyBlck.root, forkyBlck) - await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, - Opt.some(blobs), - maybeFinalized = maybeFinalized) else: await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, Opt.none(BlobSidecars), @@ -1641,6 +1640,8 @@ proc onSlotEnd(node: BeaconNode, slot: Slot) {.async.} = .pruneAfterFinalization( node.dag.finalizedHead.slot.epoch() ) + node.processor.blobQuarantine[].pruneAfterFinalization( + node.dag.finalizedHead.slot.epoch()) # Delay part of pruning until latency critical duties are done. # The other part of pruning, `pruneBlocksDAG`, is done eagerly. diff --git a/beacon_chain/sync/request_manager.nim b/beacon_chain/sync/request_manager.nim index 7ef0a61b45..27dd3b25a3 100644 --- a/beacon_chain/sync/request_manager.nim +++ b/beacon_chain/sync/request_manager.nim @@ -7,7 +7,7 @@ {.push raises: [].} -import chronos, chronicles +import std/[sets, sequtils], chronos, chronicles import ssz_serialization/types import ../spec/[forks, network, peerdas_helpers], @@ -63,6 +63,10 @@ type InhibitFn = proc: bool {.gcsafe, raises: [].} + BlobResponseRecord = object + block_root: Eth2Digest + sidecar: ref BlobSidecar + RequestManager* = object network*: Eth2Node supernode*: bool @@ -134,37 +138,35 @@ func cmpSidecarIdentifier(x: BlobIdentifier | DataColumnIdentifier, func cmpColumnIndex(x: ColumnIndex, y: ref DataColumnSidecar): int = cmp(x, y[].index) -func checkResponseSanity(idList: seq[BlobIdentifier], - blobs: openArray[ref BlobSidecar]): bool = +func checkResponseSanity( + idents: openArray[BlobIdentifier], + blobs: openArray[ref BlobSidecar] +): Opt[seq[BlobResponseRecord]] = # Cannot respond more than what I have asked - if blobs.len > idList.len: - return false - var i = 0 - while i < blobs.len: + if len(blobs) > len(idents): + return Opt.none(seq[BlobResponseRecord]) + + var + checks = idents.toHashSet() + records: seq[BlobResponseRecord] + + for sidecar in blobs.items(): let - block_root = - hash_tree_root(blobs[i][].signed_block_header.message) - idListKey = binarySearch(idList, blobs[i], cmpSidecarIdentifier) + slot = sidecar[].signed_block_header.message.slot + block_root = hash_tree_root(sidecar[].signed_block_header.message) + sidecarIdent = + BlobIdentifier(block_root: block_root, index: sidecar[].index) - # Verify the block root - if idList[idListKey].block_root != block_root: - return false + if checks.missingOrExcl(sidecarIdent): + return Opt.none(seq[BlobResponseRecord]) # Verify inclusion proof - blobs[i][].verify_blob_sidecar_inclusion_proof().isOkOr: - return false - inc i - true + sidecar[].verify_blob_sidecar_inclusion_proof().isOkOr: + return Opt.none(seq[BlobResponseRecord]) -func checkResponseSubset(idList: seq[BlobIdentifier], - blobs: openArray[ref BlobSidecar]): bool = - ## Clients MUST respond with at least one sidecar, if they have it. - ## Clients MAY limit the number of blocks and sidecars in the response. - ## https://github.com/ethereum/consensus-specs/blob/v1.5.0-beta.2/specs/deneb/p2p-interface.md#blobsidecarsbyroot-v1 - for blb in blobs: - if binarySearch(idList, blb, cmpSidecarIdentifier) == -1: - return false - true + records.add(BlobResponseRecord(block_root: block_root, sidecar: sidecar)) + + Opt.some(records) func checkColumnResponse(idList: seq[DataColumnsByRootIdentifier], columns: openArray[ref DataColumnSidecar]): bool = @@ -275,35 +277,28 @@ proc fetchBlobsFromNetwork(self: RequestManager, if blobs.isOk: var ublobs = blobs.get().asSeq() - ublobs.sort(cmpSidecarIndexes) - if not checkResponseSubset(idList, ublobs): - debug "Response to blobs by root is not a subset", - peer = peer, blobs = shortLog(idList), ublobs = len(ublobs) + let records = checkResponseSanity(idList, ublobs).valueOr: + debug "Response to blobs by root is incorrect", + peer = peer, blobs = shortLog(idList), ublobs = len(ublobs) peer.updateScore(PeerScoreBadResponse) return - if not checkResponseSanity(idList, ublobs): - debug "Response to blobs by root have erroneous block root", - peer = peer, blobs = shortLog(idList), ublobs = len(ublobs) - peer.updateScore(PeerScoreBadResponse) - return + for b in records: + self.blobQuarantine[].put(b.block_root, b.sidecar) - for b in ublobs: - self.blobQuarantine[].put(b) var curRoot: Eth2Digest - for b in ublobs: - let block_root = hash_tree_root(b.signed_block_header.message) - if block_root != curRoot: - curRoot = block_root + for record in records: + if record.block_root != curRoot: + curRoot = record.block_root if (let o = self.quarantine[].popBlobless(curRoot); o.isSome): - let b = o.unsafeGet() - discard await self.blockVerifier(b, false) + let blck = o.unsafeGet() + discard await self.blockVerifier(blck, false) # TODO: # If appropriate, return a VerifierError.InvalidBlob from # verification, check for it here, and penalize the peer accordingly else: debug "Blobs by root request failed", - peer = peer, blobs = shortLog(idList), err = blobs.error() + peer = peer, blobs = shortLog(idList), err = blobs.error() peer.updateScore(PeerScoreNoValues) finally: @@ -461,7 +456,7 @@ proc getMissingBlobs(rman: RequestManager): seq[BlobIdentifier] = waitDur = TimeDiff(nanoseconds: BLOB_GOSSIP_WAIT_TIME_NS) var - fetches: seq[BlobIdentifier] + idents: seq[BlobIdentifier] ready: seq[Eth2Digest] for blobless in rman.quarantine[].peekBlobless(): withBlck(blobless): @@ -471,28 +466,32 @@ proc getMissingBlobs(rman: RequestManager): seq[BlobIdentifier] = debug "Not handling missing blobs early in slot" continue - if not rman.blobQuarantine[].hasBlobs(forkyBlck): - let missing = rman.blobQuarantine[].blobFetchRecord(forkyBlck) - if len(missing.indices) == 0: - warn "quarantine missing blobs, but missing indices is empty", - blk=blobless.root, - commitments=len(forkyBlck.message.body.blob_kzg_commitments) - for idx in missing.indices: - let id = BlobIdentifier(block_root: blobless.root, index: idx) - if id notin fetches: - fetches.add(id) + let + commitmentsCount = len(forkyBlck.message.body.blob_kzg_commitments) + missing = + rman.blobQuarantine[].fetchMissingSidecars(blobless.root, forkyBlck) + + if len(missing) > 0: + for ident in missing: + idents.add(ident) else: - # this is a programming error should it occur. - warn "missing blob handler found blobless block with all blobs", - blk=blobless.root, - commitments=len(forkyBlck.message.body.blob_kzg_commitments) - ready.add(blobless.root) + if commitmentsCount == 0: + # this is a programming error should it occur. + warn "missing blob handler found blobless block with all blobs", + blk = blobless.root, + commitments = len(forkyBlck.message.body.blob_kzg_commitments) + ready.add(blobless.root) + else: + # This should not happen either... + warn "quarantine missing blobs, but missing indices is empty", + blk = blobless.root, + commitments = len(forkyBlck.message.body.blob_kzg_commitments) for root in ready: let blobless = rman.quarantine[].popBlobless(root).valueOr: continue discard rman.blockVerifier(blobless, false) - fetches + idents proc requestManagerBlobLoop( rman: RequestManager) {.async: (raises: [CancelledError]).} = @@ -529,7 +528,7 @@ proc requestManagerBlobLoop( discard blockRoots.pop() continue debug "Loaded orphaned blob from storage", blobId - rman.blobQuarantine[].put(blob_sidecar) + rman.blobQuarantine[].put(curRoot, blob_sidecar) var verifiers = newSeqOfCap[ Future[Result[void, VerifierError]] .Raising([CancelledError])](blockRoots.len) diff --git a/tests/all_tests.nim b/tests/all_tests.nim index 0f70be7866..68ad3716bb 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -63,6 +63,7 @@ import # Unit test ./test_beacon_validators, ./test_beacon_chain_file, ./test_mev_calls, + ./test_quarantine, ./test_keymanager_api # currently has to run after test_remote_keystore summarizeLongTests("AllTests") diff --git a/tests/test_quarantine.nim b/tests/test_quarantine.nim new file mode 100644 index 0000000000..bd93edb38d --- /dev/null +++ b/tests/test_quarantine.nim @@ -0,0 +1,1635 @@ +# beacon_chain +# Copyright (c) 2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} +{.used.} + +import std/[strutils, sequtils], stew/endians2, + kzg4844/kzg, + unittest2, + ./testutil, + ../beacon_chain/spec/datatypes/[deneb, electra, fulu], + ../beacon_chain/spec/[presets, helpers], + ../beacon_chain/consensus_object_pools/blob_quarantine + +func genBlockRoot(index: int): Eth2Digest = + var res: Eth2Digest + let tmp = uint64(index).toBytesLE() + copyMem(addr res.data[0], unsafeAddr tmp[0], sizeof(uint64)) + res + +func genKzgCommitment(index: int): KzgCommitment = + var res: KzgCommitment + let tmp = uint64(index).toBytesLE() + copyMem(addr res.bytes[0], unsafeAddr tmp[0], sizeof(uint64)) + res + +func genBlobSidecar( + index: int, + slot: int, + kzg_commitment: int, + proposer_index: int +): BlobSidecar = + BlobSidecar( + index: BlobIndex(index), + kzg_commitment: genKzgCommitment(kzg_commitment), + signed_block_header: SignedBeaconBlockHeader( + message: BeaconBlockHeader( + slot: Slot(slot), + proposer_index: uint64(proposer_index)))) + +func genDataColumnSidecar( + index: int, + slot: int, + proposer_index: int +): DataColumnSidecar = + DataColumnSidecar( + index: ColumnIndex(index), + signed_block_header: SignedBeaconBlockHeader( + message: BeaconBlockHeader( + slot: Slot(slot), + proposer_index: uint64(proposer_index)))) + +func genDenebSignedBeaconBlock( + blockRoot: Eth2Digest, + sidecars: openArray[ref BlobSidecar] +): deneb.SignedBeaconBlock = + var res: seq[KzgCommitment] + for sidecar in sidecars: + res.add(sidecar[].kzg_commitment) + deneb.SignedBeaconBlock( + message: deneb.BeaconBlock( + body: deneb.BeaconBlockBody(blob_kzg_commitments: KzgCommitments(res))), + root: blockRoot) + +func genElectraSignedBeaconBlock( + blockRoot: Eth2Digest, + sidecars: openArray[ref BlobSidecar] +): electra.SignedBeaconBlock = + var res: seq[KzgCommitment] + for sidecar in sidecars: + res.add(sidecar[].kzg_commitment) + electra.SignedBeaconBlock( + message: electra.BeaconBlock( + body: electra.BeaconBlockBody(blob_kzg_commitments: KzgCommitments(res))), + root: blockRoot) + +func genFuluSignedBeaconBlock( + blockRoot: Eth2Digest, + commitments: openArray[KzgCommitment] +): fulu.SignedBeaconBlock = + var res = @commitments + fulu.SignedBeaconBlock( + message: fulu.BeaconBlock( + body: fulu.BeaconBlockBody(blob_kzg_commitments: KzgCommitments(res))), + root: blockRoot) + +func compareSidecars( + a, b: openArray[ref BlobSidecar|ref DataColumnSidecar] +): bool = + if len(a) != len(b): + return false + if len(a) == 0: + return true + for i in 0 ..< len(a): + if cast[uint64](a[i]) != cast[uint64](b[i]): + return false + true + +func compareSidecars( + blockRoot: Eth2Digest, + a: openArray[ref BlobSidecar|ref DataColumnSidecar], + b: openArray[BlobIdentifier|DataColumnIdentifier] +): bool = + if len(a) != len(b): + return false + if len(a) == 0: + return true + for i in 0 ..< len(a): + if (a[i][].index != b[i].index) or (b[i].block_root != blockRoot): + return false + true + +func compareIdentifiers( + a, b: openArray[DataColumnIdentifier]): bool = + if len(a) != len(b): + return false + if len(a) == 0: + return true + for i in 0 ..< len(a): + if (a[i].block_root != b[i].block_root) or (a[i].index != b[i].index): + return false + true + +func supernodeColumns(): seq[ColumnIndex] = + var res: seq[ColumnIndex] + for i in 0 ..< 128: + res.add(ColumnIndex(i)) + res + +suite "BlobQuarantine data structure test suite " & preset(): + setup: + let cfg = defaultRuntimeConfig + + test "put()/hasSidecar(index, slot, proposer_index)/remove() test": + var bq = BlobQuarantine.init(cfg, nil) + let + broot1 = genBlockRoot(1) + broot2 = genBlockRoot(2) + broot3 = genBlockRoot(3) + broot4 = genBlockRoot(4) + broot5 = genBlockRoot(5) + sidecar1 = + newClone(genBlobSidecar(index = 0, slot = 1, 1, proposer_index = 5)) + sidecar2 = + newClone(genBlobSidecar(index = 1, slot = 1, 2, proposer_index = 5)) + sidecar3 = + newClone(genBlobSidecar(index = 2, slot = 1, 3, proposer_index = 5)) + sidecar4 = + newClone(genBlobSidecar(index = 4, slot = 2, 4, proposer_index = 6)) + sidecar5 = + newClone(genBlobSidecar(index = 5, slot = 3, 5, proposer_index = 7)) + sidecar6 = + newClone(genBlobSidecar(index = 6, slot = 3, 6, proposer_index = 8)) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(0)) == false + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(1)) == false + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(2)) == false + bq.hasSidecar(broot2, Slot(2), uint64(5), BlobIndex(4)) == false + bq.hasSidecar(broot3, Slot(3), uint64(5), BlobIndex(5)) == false + bq.hasSidecar(broot4, Slot(3), uint64(5), BlobIndex(6)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), BlobIndex(3)) == false + + bq.put(broot1, sidecar1) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(1)) == false + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(2)) == false + bq.hasSidecar(broot2, Slot(2), uint64(6), BlobIndex(4)) == false + bq.hasSidecar(broot3, Slot(3), uint64(7), BlobIndex(5)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), BlobIndex(6)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), BlobIndex(3)) == false + + bq.put(broot1, sidecar2) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(1)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(2)) == false + bq.hasSidecar(broot2, Slot(2), uint64(6), BlobIndex(4)) == false + bq.hasSidecar(broot3, Slot(3), uint64(7), BlobIndex(5)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), BlobIndex(6)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), BlobIndex(3)) == false + + bq.put(broot1, sidecar3) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(1)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(2)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), BlobIndex(4)) == false + bq.hasSidecar(broot3, Slot(3), uint64(7), BlobIndex(5)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), BlobIndex(6)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), BlobIndex(3)) == false + + bq.put(broot2, sidecar4) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(1)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(2)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), BlobIndex(4)) == true + bq.hasSidecar(broot3, Slot(3), uint64(7), BlobIndex(5)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), BlobIndex(6)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), BlobIndex(3)) == false + + bq.put(broot3, sidecar5) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(1)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(2)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), BlobIndex(4)) == true + bq.hasSidecar(broot3, Slot(3), uint64(7), BlobIndex(5)) == true + bq.hasSidecar(broot4, Slot(3), uint64(8), BlobIndex(6)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), BlobIndex(3)) == false + + bq.put(broot4, sidecar6) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(1)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(2)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), BlobIndex(4)) == true + bq.hasSidecar(broot3, Slot(3), uint64(7), BlobIndex(5)) == true + bq.hasSidecar(broot4, Slot(3), uint64(8), BlobIndex(6)) == true + bq.hasSidecar(broot5, Slot(10), uint64(100), BlobIndex(3)) == false + + bq.remove(broot4) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(1)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(2)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), BlobIndex(4)) == true + bq.hasSidecar(broot3, Slot(3), uint64(7), BlobIndex(5)) == true + bq.hasSidecar(broot4, Slot(3), uint64(8), BlobIndex(6)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), BlobIndex(3)) == false + + bq.remove(broot3) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(1)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(2)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), BlobIndex(4)) == true + bq.hasSidecar(broot3, Slot(3), uint64(7), BlobIndex(5)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), BlobIndex(6)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), BlobIndex(3)) == false + + bq.remove(broot2) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(1)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(2)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), BlobIndex(4)) == false + bq.hasSidecar(broot3, Slot(3), uint64(7), BlobIndex(5)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), BlobIndex(6)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), BlobIndex(3)) == false + + bq.remove(broot1) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(0)) == false + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(1)) == false + bq.hasSidecar(broot1, Slot(1), uint64(5), BlobIndex(2)) == false + bq.hasSidecar(broot2, Slot(2), uint64(6), BlobIndex(4)) == false + bq.hasSidecar(broot3, Slot(3), uint64(7), BlobIndex(5)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), BlobIndex(6)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), BlobIndex(3)) == false + len(bq) == 0 + + test "put(sidecar)/put([sidecars])/hasSidecars/popSidecars/remove() test": + var bq = BlobQuarantine.init(cfg, nil) + let + broot1 = genBlockRoot(1) + broot2 = genBlockRoot(2) + sidecars1 = + block: + var res: seq[ref BlobSidecar] + for i in 0 ..< cfg.MAX_BLOBS_PER_BLOCK_ELECTRA: + res.add(newClone(genBlobSidecar(index = int(i), slot = 1, + 1 + int(i), proposer_index = 5))) + res + sidecars2 = + block: + var res: seq[ref BlobSidecar] + for i in 0 ..< cfg.MAX_BLOBS_PER_BLOCK_ELECTRA: + res.add(newClone(genBlobSidecar(index = int(i), slot = 1, + 1 + int(i), proposer_index = 50))) + res + denebBlock = genDenebSignedBeaconBlock(broot1, sidecars1) + electraBlock = genElectraSignedBeaconBlock(broot2, sidecars2) + + check: + bq.hasSidecars(denebBlock) == false + bq.popSidecars(denebBlock).isNone() == true + bq.hasSidecars(electraBlock) == false + bq.popSidecars(electraBlock).isNone() == true + + bq.put(broot1, sidecars1) + + for index in 0 ..< len(sidecars2): + if index mod 2 != 1: + bq.put(broot2, sidecars2[index]) + + check: + bq.hasSidecars(denebBlock) == true + bq.hasSidecars(electraBlock) == false + bq.popSidecars(electraBlock).isNone() == true + let dres = bq.popSidecars(denebBlock) + check: + dres.isOk() + compareSidecars(dres.get(), sidecars1) == true + + bq.put(broot2, sidecars2[1]) + check: + bq.hasSidecars(electraBlock) == false + bq.popSidecars(electraBlock).isNone() == true + + bq.put(broot2, sidecars2[3]) + check: + bq.hasSidecars(electraBlock) == false + bq.popSidecars(electraBlock).isNone() == true + + bq.put(broot2, sidecars2[5]) + check: + bq.hasSidecars(electraBlock) == false + bq.popSidecars(electraBlock).isNone() == true + + bq.put(broot2, sidecars2[7]) + check: + bq.hasSidecars(electraBlock) == true + let eres = bq.popSidecars(electraBlock) + check: + eres.isOk() + compareSidecars(eres.get(), sidecars2) == true + + bq.remove(broot1) + bq.remove(broot2) + check: + len(bq) == 0 + + test "put()/fetchMissingSidecars/remove test": + var bq = BlobQuarantine.init(cfg, nil) + let + broot1 = genBlockRoot(1) + broot2 = genBlockRoot(2) + sidecars1 = + block: + var res: seq[ref BlobSidecar] + for i in 0 ..< cfg.MAX_BLOBS_PER_BLOCK_ELECTRA: + res.add(newClone(genBlobSidecar(index = int(i), slot = 1, + 1 + int(i), proposer_index = 5))) + res + sidecars2 = + block: + var res: seq[ref BlobSidecar] + for i in 0 ..< cfg.MAX_BLOBS_PER_BLOCK_ELECTRA: + res.add(newClone(genBlobSidecar(index = int(i), slot = 1, + 1 + int(i), proposer_index = 50))) + res + denebBlock = genDenebSignedBeaconBlock(broot1, sidecars1) + electraBlock = genElectraSignedBeaconBlock(broot2, sidecars2) + + for i in 0 ..< len(sidecars1) + 1: + let + missing1 = bq.fetchMissingSidecars(broot1, denebBlock) + missing2 = bq.fetchMissingSidecars(broot2, electraBlock) + + check: + compareSidecars( + broot1, + sidecars1.toOpenArray(i, len(sidecars1) - 1), missing1) == true + compareSidecars( + broot2, + sidecars2.toOpenArray(i, len(sidecars2) - 1), missing2) == true + + if i >= len(sidecars1): + break + + bq.put(broot1, sidecars1[i]) + bq.put(broot2, sidecars2[i]) + + bq.remove(broot1) + bq.remove(broot2) + check len(bq) == 0 + + test "popSidecars()/hasSidecars() return []/true on block without blobs": + var + bq = BlobQuarantine.init(cfg, nil) + let + blockRoot1 = genBlockRoot(100) + blockRoot2 = genBlockRoot(5337) + blockRoot3 = genBlockRoot(191925) + blockRoot4 = genBlockRoot(1294967295) + denebBlock1 = genDenebSignedBeaconBlock(blockRoot1, []) + denebBlock2 = genDenebSignedBeaconBlock(blockRoot2, []) + electraBlock1 = genElectraSignedBeaconBlock(blockRoot3, []) + electraBlock2 = genElectraSignedBeaconBlock(blockRoot4, []) + check: + bq.hasSidecars(denebBlock1.root, denebBlock1) == true + bq.hasSidecars(denebBlock2.root, denebBlock2) == true + bq.hasSidecars(electraBlock1.root, electraBlock1) == true + bq.hasSidecars(electraBlock2.root, electraBlock2) == true + + let + res1 = bq.popSidecars(denebBlock1.root, denebBlock1) + res2 = bq.popSidecars(denebBlock2.root, denebBlock2) + res3 = bq.popSidecars(electraBlock1.root, electraBlock1) + res4 = bq.popSidecars(electraBlock2.root, electraBlock2) + + check: + res1.isOk() + len(res1.get()) == 0 + res2.isOk() + len(res2.get()) == 0 + res3.isOk() + len(res3.get()) == 0 + res4.isOk() + len(res4.get()) == 0 + + test "overfill protection test": + var + bq = BlobQuarantine.init(cfg, nil) + sidecars: seq[tuple[sidecar: ref BlobSidecar, blockRoot: Eth2Digest]] + + let maxSidecars = int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA * SLOTS_PER_EPOCH) * 3 + for i in 0 ..< maxSidecars: + let + index = i mod int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) + slot = i div int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) + 100 + blockRoot = genBlockRoot(slot) + sidecar = newClone(genBlobSidecar(index, slot, i, proposer_index = i)) + sidecars.add((sidecar, blockRoot)) + + for item in sidecars: + bq.put(item.blockRoot, item.sidecar) + + # put(sidecar) test + + check len(bq) == maxSidecars + + for i in 0 ..< int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA): + check: + bq.hasSidecar( + blockRoot = + genBlockRoot( + int(sidecars[i].sidecar[].signed_block_header.message.slot)), + slot = + sidecars[i].sidecar[].signed_block_header.message.slot, + proposer_index = + sidecars[i].sidecar[].signed_block_header.message.proposer_index, + index = sidecars[i].sidecar[].index + ) == true + + let + sidecar = newClone(genBlobSidecar(index = 0, slot = 10000, 100000, + proposer_index = 1000000)) + blockRoot = genBlockRoot(10000) + check: + bq.hasSidecar(blockRoot = blockRoot, slot = Slot(10000), + proposer_index = 1000000'u64, index = BlobIndex(0)) == false + bq.put(blockRoot, sidecar) + check: + len(bq) == (len(sidecars) - int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) + 1) + bq.hasSidecar(blockRoot = blockRoot, slot = Slot(10000), + proposer_index = 1000000'u64, index = BlobIndex(0)) == true + + for i in 0 ..< int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA): + check: + bq.hasSidecar( + blockRoot = + genBlockRoot( + int(sidecars[i].sidecar[].signed_block_header.message.slot)), + slot = + sidecars[i].sidecar[].signed_block_header.message.slot, + proposer_index = + sidecars[i].sidecar[].signed_block_header.message.proposer_index, + index = sidecars[i].sidecar[].index + ) == false + + # put(openArray[sidecar]) test + + let + msidecars = + block: + var res: seq[ref BlobSidecar] + for i in 0 ..< int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA): + let sidecar = + newClone(genBlobSidecar(index = i, slot = 100_000, 200000, + proposer_index = 2000000)) + res.add(sidecar) + res + mblockRoot = genBlockRoot(20000) + + check: + len(bq) == (len(sidecars) - int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) + 1) + + let beforeLength = len(bq) + + for s in msidecars: + check: + bq.hasSidecar(mblockRoot, + s.signed_block_header.message.slot, + s.signed_block_header.message.proposer_index, + s.index) == false + + bq.put(mblockRoot, msidecars) + check len(bq) == beforeLength + + for s in msidecars: + check: + bq.hasSidecar(mblockRoot, + s.signed_block_header.message.slot, + s.signed_block_header.message.proposer_index, + s.index) == true + + for i in 0 ..< int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA): + let j = int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) + i + check: + bq.hasSidecar( + blockRoot = + genBlockRoot( + int(sidecars[j].sidecar[].signed_block_header.message.slot)), + slot = + sidecars[j].sidecar[].signed_block_header.message.slot, + proposer_index = + sidecars[j].sidecar[].signed_block_header.message.proposer_index, + index = sidecars[j].sidecar[].index + ) == false + + test "put() duplicate items should not affect counters": + var + bq = BlobQuarantine.init(cfg, nil) + sidecars1: seq[ref BlobSidecar] + sidecars1d: seq[ref BlobSidecar] + sidecars2: seq[ref BlobSidecar] + sidecars2d: seq[ref BlobSidecar] + + for index in 0 ..< cfg.MAX_BLOBS_PER_BLOCK_ELECTRA: + let + sidecar1 = newClone(genBlobSidecar(int(index), 1, int(index), 64)) + sidecar1d = newClone(genBlobSidecar(int(index), 1, int(index), 64)) + sidecar2 = newClone(genBlobSidecar(int(index), 2, 50 + int(index), 65)) + sidecar2d = newClone(genBlobSidecar(int(index), 2, 50 + int(index), 65)) + sidecars1.add(sidecar1) + sidecars1d.add(sidecar1d) + sidecars2.add(sidecar2) + sidecars2d.add(sidecar2d) + + let + broot1 = genBlockRoot(100) + broot2 = genBlockRoot(200) + + electraBlock1 = genElectraSignedBeaconBlock(broot1, sidecars1) + electraBlock2 = genElectraSignedBeaconBlock(broot2, sidecars2) + + check: + len(bq) == 0 + len(bq.fetchMissingSidecars(broot1, electraBlock1)) == + int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) + len(bq.fetchMissingSidecars(broot2, electraBlock2)) == + int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) + + for index in 0 ..< int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA): + bq.put(broot1, sidecars1[index]) + check: + len(bq) == (index + 1) + len(bq.fetchMissingSidecars(broot1, electraBlock1)) == + int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) - (index + 1) + bq.put(broot1, sidecars1d[index]) + check: + len(bq) == (index + 1) + len(bq.fetchMissingSidecars(broot1, electraBlock1)) == + int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) - (index + 1) + + for index in 0 ..< int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA): + bq.put(broot2, sidecars2[index]) + check: + len(bq) == int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) + (index + 1) + len(bq.fetchMissingSidecars(broot2, electraBlock2)) == + int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) - (index + 1) + bq.put(broot2, sidecars2d[index]) + check: + len(bq) == int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) + (index + 1) + len(bq.fetchMissingSidecars(broot2, electraBlock2)) == + int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) - (index + 1) + + bq.remove(broot2) + check len(bq) == int(cfg.MAX_BLOBS_PER_BLOCK_ELECTRA) + bq.remove(broot1) + check len(bq) == 0 + + test "pruneAfterFinalization() test": + const TestVectors = [ + (root: 1, slot: 1, kzg: 1, index: 0, proposer_index: 20), + (root: 1, slot: 1, kzg: 2, index: 1, proposer_index: 20), + (root: 1, slot: 1, kzg: 3, index: 2, proposer_index: 20), + (root: 1, slot: 1, kzg: 4, index: 3, proposer_index: 20), + (root: 1, slot: 1, kzg: 5, index: 4, proposer_index: 20), + (root: 2, slot: 32, kzg: 6, index: 0, proposer_index: 21), + (root: 2, slot: 32, kzg: 7, index: 1, proposer_index: 21), + (root: 2, slot: 32, kzg: 8, index: 2, proposer_index: 21), + (root: 3, slot: 33, kzg: 9, index: 3, proposer_index: 22), + (root: 3, slot: 33, kzg: 10, index: 4, proposer_index: 22), + (root: 4, slot: 63, kzg: 11, index: 5, proposer_index: 23), + (root: 5, slot: 64, kzg: 12, index: 0, proposer_index: 24), + (root: 5, slot: 64, kzg: 13, index: 1, proposer_index: 24), + (root: 5, slot: 64, kzg: 14, index: 2, proposer_index: 24), + (root: 6, slot: 65, kzg: 15, index: 0, proposer_index: 25), + (root: 6, slot: 65, kzg: 16, index: 1, proposer_index: 25), + (root: 7, slot: 67, kzg: 17, index: 0, proposer_index: 26), + (root: 7, slot: 67, kzg: 18, index: 1, proposer_index: 26), + (root: 8, slot: 95, kzg: 19, index: 0, proposer_index: 27), + (root: 8, slot: 95, kzg: 20, index: 1, proposer_index: 27), + (root: 8, slot: 95, kzg: 21, index: 2, proposer_index: 27), + (root: 8, slot: 95, kzg: 22, index: 3, proposer_index: 27), + (root: 8, slot: 95, kzg: 23, index: 4, proposer_index: 27), + (root: 9, slot: 96, kzg: 24, index: 0, proposer_index: 28), + (root: 9, slot: 96, kzg: 25, index: 1, proposer_index: 28), + (root: 9, slot: 96, kzg: 26, index: 2, proposer_index: 28), + (root: 9, slot: 96, kzg: 27, index: 3, proposer_index: 28), + (root: 9, slot: 96, kzg: 28, index: 4, proposer_index: 28), + (root: 9, slot: 96, kzg: 29, index: 5, proposer_index: 28), + (root: 9, slot: 96, kzg: 30, index: 6, proposer_index: 28), + (root: 9, slot: 96, kzg: 31, index: 7, proposer_index: 28), + (root: 9, slot: 96, kzg: 32, index: 8, proposer_index: 28), + (root: 10, slot: 127, kzg: 33, index: 0, proposer_index: 29), + (root: 10, slot: 127, kzg: 34, index: 1, proposer_index: 29), + (root: 10, slot: 127, kzg: 35, index: 2, proposer_index: 29) + ] + + var bq = BlobQuarantine.init(cfg, nil) + for item in TestVectors: + let sidecar = + newClone( + genBlobSidecar(index = item.index, slot = item.slot, item.kzg, + proposer_index = item.proposer_index)) + bq.put(genBlockRoot(item.root), sidecar) + + check: + len(bq) == len(TestVectors) + + for item in TestVectors: + check: + bq.hasSidecar( + genBlockRoot(item.root), Slot(item.slot), + uint64(item.proposer_index), BlobIndex(item.index)) == true + + bq.pruneAfterFinalization(Epoch(1)) + check: + len(bq) == len(TestVectors) - 5 + + for item in TestVectors: + let res = + if item.root == 1: + false + else: + true + check: + bq.hasSidecar( + genBlockRoot(item.root), Slot(item.slot), + uint64(item.proposer_index), BlobIndex(item.index)) == res + + bq.pruneAfterFinalization(Epoch(2)) + check: + len(bq) == len(TestVectors) - 5 - 6 + + for item in TestVectors: + let res = + if item.root in [1, 2, 3, 4]: + false + else: + true + check: + bq.hasSidecar( + genBlockRoot(item.root), Slot(item.slot), + uint64(item.proposer_index), BlobIndex(item.index)) == res + + bq.pruneAfterFinalization(Epoch(3)) + check: + len(bq) == len(TestVectors) - 5 - 6 - 12 + + for item in TestVectors: + let res = + if item.root in [1, 2, 3, 4, 5, 6, 7, 8]: + false + else: + true + check: + bq.hasSidecar( + genBlockRoot(item.root), Slot(item.slot), + uint64(item.proposer_index), BlobIndex(item.index)) == res + + bq.pruneAfterFinalization(Epoch(4)) + check: + len(bq) == 0 + + for item in TestVectors: + check: + bq.hasSidecar( + genBlockRoot(item.root), Slot(item.slot), + uint64(item.proposer_index), BlobIndex(item.index)) == false + +suite "ColumnQuarantine data structure test suite " & preset(): + setup: + let cfg {.used.} = defaultRuntimeConfig + + test "ColumnMap test": + # Filling columns of different sizes with all bits [8, 128) + for columnSize in 8 .. 128: + let + columnsCount = 128 div columnSize + lastColumnSize = 128 mod columnSize + + for i in 0 ..< columnsCount: + let + start = i * columnSize + finish = start + columnSize + var + columns: seq[ColumnIndex] + numbers: seq[int] + for k in start ..< finish: + columns.add(ColumnIndex(k)) + numbers.add(k) + + check: + $ColumnMap.init(columns) == + "[" & $ numbers.mapIt($it).join(", ") & "]" + + if lastColumnSize > 0: + let + start = columnsCount * columnSize + finish = start + lastColumnSize + var + columns: seq[ColumnIndex] + numbers: seq[int] + for k in start ..< finish: + columns.add(ColumnIndex(k)) + numbers.add(k) + + check: + $ColumnMap.init(columns) == + "[" & $ numbers.mapIt($it).join(", ") & "]" + + # Verify `and` operation is correct + const TestVectors = [ + ( + [1, 2, 3, 4, 5, 6, 7, 8], + [5, 6, 7, 8, 9, 10, 11, 12], + "[5, 6, 7, 8]" + ), + ( + [56, 57, 58, 59, 60, 61, 62, 63], + [60, 61, 62, 63, 64, 65, 66, 67], + "[60, 61, 62, 63]" + ), + ( + [1, 5, 10, 15, 20, 25, 64, 65], + [1, 5, 6, 7, 8, 9, 64, 65], + "[1, 5, 64, 65]" + ), + ( + [60, 61, 62, 63, 124, 125, 126, 127], + [60, 61, 62, 63, 124, 125, 126, 127], + "[60, 61, 62, 63, 124, 125, 126, 127]" + ), + ( + [0, 1, 63, 64, 65, 93, 126, 127], + [0, 2, 63, 64, 67, 94, 126, 127], + "[0, 63, 64, 126, 127]" + ) + ] + + for vector in TestVectors: + let + map1 = ColumnMap.init(vector[0].mapIt(ColumnIndex(it))) + map2 = ColumnMap.init(vector[1].mapIt(ColumnIndex(it))) + check: + $(map1 and map2) == vector[2] + + for vector in TestVectors: + let + map1 = ColumnMap.init(vector[0].mapIt(ColumnIndex(it))) + map2 = ColumnMap.init(vector[1].mapIt(ColumnIndex(it))) + map3 = map1 and map2 + + check: + map1.items().toSeq().mapIt($int(it)).join(", ") == + vector[0].mapIt($it).join(", ") + map2.items().toSeq().mapIt($int(it)).join(", ") == + vector[1].mapIt($it).join(", ") + "[" & map3.items().toSeq().mapIt($int(it)).join(", ") & "]" == + vector[2] + + var columns: seq[ColumnIndex] + for i in 0 ..< NUMBER_OF_COLUMNS: + columns.add(ColumnIndex(i)) + let map = ColumnMap.init(columns) + check: + map.items().toSeq().mapIt($int(it)).join(", ") == + columns.mapIt($it).join(", ") + + test "put()/hasSidecar(index, slot, proposer_index)/remove() test": + let custodyColumns = + [0, 31, 32, 63, 64, 95, 96, 127].mapIt(ColumnIndex(it)) + var bq = ColumnQuarantine.init(cfg, custodyColumns, nil) + let + broot1 = genBlockRoot(1) + broot2 = genBlockRoot(2) + broot3 = genBlockRoot(3) + broot4 = genBlockRoot(4) + broot5 = genBlockRoot(5) + sidecar1 = + newClone(genDataColumnSidecar( + index = 0, slot = 1, proposer_index = 5)) + sidecar2 = + newClone(genDataColumnSidecar( + index = 31, slot = 1, proposer_index = 5)) + sidecar3 = + newClone(genDataColumnSidecar( + index = 32, slot = 1, proposer_index = 5)) + sidecar4 = + newClone(genDataColumnSidecar( + index = 127, slot = 2, proposer_index = 6)) + sidecar5 = + newClone(genDataColumnSidecar( + index = 0, slot = 3, proposer_index = 7)) + sidecar6 = + newClone(genDataColumnSidecar( + index = 31, slot = 3, proposer_index = 8)) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(0)) == false + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(31)) == false + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(32)) == false + bq.hasSidecar(broot2, Slot(2), uint64(5), ColumnIndex(127)) == false + bq.hasSidecar(broot3, Slot(3), uint64(5), ColumnIndex(0)) == false + bq.hasSidecar(broot4, Slot(3), uint64(5), ColumnIndex(31)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), ColumnIndex(3)) == false + + bq.put(broot1, sidecar1) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(31)) == false + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(32)) == false + bq.hasSidecar(broot2, Slot(2), uint64(6), ColumnIndex(127)) == false + bq.hasSidecar(broot3, Slot(3), uint64(7), ColumnIndex(0)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), ColumnIndex(31)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), ColumnIndex(3)) == false + + bq.put(broot1, sidecar2) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(31)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(32)) == false + bq.hasSidecar(broot2, Slot(2), uint64(6), ColumnIndex(127)) == false + bq.hasSidecar(broot3, Slot(3), uint64(7), ColumnIndex(0)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), ColumnIndex(31)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), ColumnIndex(3)) == false + + bq.put(broot1, sidecar3) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(31)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(32)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), ColumnIndex(127)) == false + bq.hasSidecar(broot3, Slot(3), uint64(7), ColumnIndex(0)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), ColumnIndex(31)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), ColumnIndex(3)) == false + + bq.put(broot2, sidecar4) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(31)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(32)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), ColumnIndex(127)) == true + bq.hasSidecar(broot3, Slot(3), uint64(7), ColumnIndex(0)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), ColumnIndex(31)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), ColumnIndex(3)) == false + + bq.put(broot3, sidecar5) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(31)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(32)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), ColumnIndex(127)) == true + bq.hasSidecar(broot3, Slot(3), uint64(7), ColumnIndex(0)) == true + bq.hasSidecar(broot4, Slot(3), uint64(8), ColumnIndex(31)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), ColumnIndex(3)) == false + + bq.put(broot4, sidecar6) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(31)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(32)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), ColumnIndex(127)) == true + bq.hasSidecar(broot3, Slot(3), uint64(7), ColumnIndex(0)) == true + bq.hasSidecar(broot4, Slot(3), uint64(8), ColumnIndex(31)) == true + bq.hasSidecar(broot5, Slot(10), uint64(100), ColumnIndex(3)) == false + + bq.remove(broot4) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(31)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(32)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), ColumnIndex(127)) == true + bq.hasSidecar(broot3, Slot(3), uint64(7), ColumnIndex(0)) == true + bq.hasSidecar(broot4, Slot(3), uint64(8), ColumnIndex(31)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), ColumnIndex(3)) == false + + bq.remove(broot3) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(31)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(32)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), ColumnIndex(127)) == true + bq.hasSidecar(broot3, Slot(3), uint64(7), ColumnIndex(0)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), ColumnIndex(31)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), ColumnIndex(3)) == false + + bq.remove(broot2) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(0)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(31)) == true + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(32)) == true + bq.hasSidecar(broot2, Slot(2), uint64(6), ColumnIndex(127)) == false + bq.hasSidecar(broot3, Slot(3), uint64(7), ColumnIndex(0)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), ColumnIndex(31)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), ColumnIndex(3)) == false + + bq.remove(broot1) + + check: + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(0)) == false + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(31)) == false + bq.hasSidecar(broot1, Slot(1), uint64(5), ColumnIndex(32)) == false + bq.hasSidecar(broot2, Slot(2), uint64(6), ColumnIndex(127)) == false + bq.hasSidecar(broot3, Slot(3), uint64(7), ColumnIndex(0)) == false + bq.hasSidecar(broot4, Slot(3), uint64(8), ColumnIndex(31)) == false + bq.hasSidecar(broot5, Slot(10), uint64(100), ColumnIndex(3)) == false + len(bq) == 0 + + test "put(sidecar)/put([sidecars])/hasSidecars/popSidecars/remove() [node] test": + let custodyColumns = + [0, 31, 32, 63, 64, 95, 96, 127].mapIt(ColumnIndex(it)) + var bq = ColumnQuarantine.init(cfg, custodyColumns, nil) + let + broot1 = genBlockRoot(1) + broot2 = genBlockRoot(2) + sidecars1 = + block: + var res: seq[ref DataColumnSidecar] + for i in 0 ..< len(custodyColumns): + res.add(newClone(genDataColumnSidecar( + index = int(custodyColumns[i]), slot = 1, proposer_index = 5))) + res + sidecars2 = + block: + var res: seq[ref DataColumnSidecar] + for i in 0 ..< len(custodyColumns): + res.add(newClone(genDataColumnSidecar( + index = int(custodyColumns[i]), slot = 1, proposer_index = 6))) + res + commitments1 = [ + genKzgCommitment(1), genKzgCommitment(2), genKzgCommitment(3) + ] + commitments2 = [ + genKzgCommitment(4), genKzgCommitment(5), genKzgCommitment(6) + ] + fuluBlock1 = genFuluSignedBeaconBlock(broot1, commitments1) + fuluBlock2 = genFuluSignedBeaconBlock(broot2, commitments2) + + check: + bq.hasSidecars(fuluBlock1) == false + bq.popSidecars(fuluBlock1).isNone() == true + bq.hasSidecars(fuluBlock2) == false + bq.popSidecars(fuluBlock2).isNone() == true + + bq.put(broot1, sidecars1) + + for index in 0 ..< len(sidecars2): + if index notin [1, 3, 5, 7]: + bq.put(broot2, sidecars2[index]) + + check: + bq.hasSidecars(fuluBlock1) == true + bq.hasSidecars(fuluBlock2) == false + bq.popSidecars(fuluBlock2).isNone() == true + let dres = bq.popSidecars(fuluBlock1) + check: + dres.isOk() + compareSidecars(dres.get(), sidecars1) == true + + bq.put(broot2, sidecars2[1]) + check: + bq.hasSidecars(fuluBlock2) == false + bq.popSidecars(fuluBlock2).isNone() == true + + bq.put(broot2, sidecars2[3]) + check: + bq.hasSidecars(fuluBlock2) == false + bq.popSidecars(fuluBlock2).isNone() == true + + bq.put(broot2, sidecars2[5]) + check: + bq.hasSidecars(fuluBlock2) == false + bq.popSidecars(fuluBlock2).isNone() == true + + bq.put(broot2, sidecars2[7]) + check: + bq.hasSidecars(fuluBlock2) == true + + let eres = bq.popSidecars(fuluBlock2) + check: + eres.isOk() + compareSidecars(eres.get(), sidecars2) == true + + bq.remove(broot1) + bq.remove(broot2) + check len(bq) == 0 + + test "put(sidecar)/put([sidecars])/hasSidecars/popSidecars/remove() [supernode] test": + let custodyColumns = supernodeColumns() + var bq = ColumnQuarantine.init(cfg, custodyColumns, nil) + let + broot1 = genBlockRoot(1) + broot2 = genBlockRoot(2) + sidecars1 = + block: + var res: seq[ref DataColumnSidecar] + for i in 0 ..< (len(custodyColumns) div 2 + 1): + res.add(newClone(genDataColumnSidecar( + index = int(custodyColumns[i]), slot = 1, proposer_index = 5))) + res + sidecars2 = + block: + var res: seq[ref DataColumnSidecar] + for i in 0 ..< (len(custodyColumns) div 2 + 1): + res.add(newClone(genDataColumnSidecar( + index = int(custodyColumns[i]), slot = 1, proposer_index = 6))) + res + commitments1 = [ + genKzgCommitment(1), genKzgCommitment(2), genKzgCommitment(3) + ] + commitments2 = [ + genKzgCommitment(4), genKzgCommitment(5), genKzgCommitment(6) + ] + fuluBlock1 = genFuluSignedBeaconBlock(broot1, commitments1) + fuluBlock2 = genFuluSignedBeaconBlock(broot2, commitments2) + + check: + bq.hasSidecars(fuluBlock1) == false + bq.popSidecars(fuluBlock1).isNone() == true + bq.hasSidecars(fuluBlock2) == false + bq.popSidecars(fuluBlock2).isNone() == true + + bq.put(broot1, sidecars1) + + for index in 0 ..< len(sidecars2): + if index notin [1, 3, 5, 7]: + bq.put(broot2, sidecars2[index]) + + check: + bq.hasSidecars(fuluBlock1) == true + bq.hasSidecars(fuluBlock2) == false + bq.popSidecars(fuluBlock2).isNone() == true + let dres = bq.popSidecars(fuluBlock1) + check: + dres.isOk() + compareSidecars(dres.get(), sidecars1) == true + + bq.put(broot2, sidecars2[1]) + check: + bq.hasSidecars(fuluBlock2) == false + bq.popSidecars(fuluBlock2).isNone() == true + + bq.put(broot2, sidecars2[3]) + check: + bq.hasSidecars(fuluBlock2) == false + bq.popSidecars(fuluBlock2).isNone() == true + + bq.put(broot2, sidecars2[5]) + check: + bq.hasSidecars(fuluBlock2) == false + bq.popSidecars(fuluBlock2).isNone() == true + + bq.put(broot2, sidecars2[7]) + check: + bq.hasSidecars(fuluBlock2) == true + + let eres = bq.popSidecars(fuluBlock2) + check: + eres.isOk() + compareSidecars(eres.get(), sidecars2) == true + + bq.remove(broot1) + bq.remove(broot2) + check len(bq) == 0 + + test "put()/fetchMissingSidecars/remove test [node]": + let + custodyColumns = + [0, 31, 32, 63, 64, 95, 96, 127].mapIt(ColumnIndex(it)) + peerCustodyColumns1 = + [63, 64, 65, 66, 95, 96, 97, 98].mapIt(ColumnIndex(it)) + peerCustodyColumns2 = + [1, 2, 3, 4, 5, 6, 7, 8].mapIt(ColumnIndex(it)) + + var bq = ColumnQuarantine.init(cfg, custodyColumns, nil) + let + broot1 = genBlockRoot(1) + broot2 = genBlockRoot(2) + expected1 = [ + @[ + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(63)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(64)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(95)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(96))], + @[ + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(63)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(64)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(95)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(96))], + @[ + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(63)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(64)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(95)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(96))], + @[ + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(63)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(64)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(95)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(96))], + @[ + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(64)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(95)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(96))], + @[ + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(95)), + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(96))], + @[ + DataColumnIdentifier(block_root: broot1, index: ColumnIndex(96))], + @[], + @[] + ] + sidecars1 = + block: + var res: seq[ref DataColumnSidecar] + for i in 0 ..< len(custodyColumns): + res.add(newClone(genDataColumnSidecar( + index = int(custodyColumns[i]), slot = 1, proposer_index = 5))) + res + sidecars2 = + block: + var res: seq[ref DataColumnSidecar] + for i in 0 ..< len(custodyColumns): + res.add(newClone(genDataColumnSidecar( + index = int(custodyColumns[i]), slot = 2, proposer_index = 50))) + res + commitments1 = [ + genKzgCommitment(1), genKzgCommitment(2), genKzgCommitment(3) + ] + commitments2 = [ + genKzgCommitment(4), genKzgCommitment(5), genKzgCommitment(6) + ] + fuluBlock1 = genFuluSignedBeaconBlock(broot1, commitments1) + fuluBlock2 = genFuluSignedBeaconBlock(broot2, commitments2) + + for i in 0 ..< len(sidecars1) + 1: + let + missing1 = bq.fetchMissingSidecars(broot1, fuluBlock1) + missing2 = bq.fetchMissingSidecars(broot2, fuluBlock2) + missing3 = bq.fetchMissingSidecars(broot1, fuluBlock1, + peerCustodyColumns1) + missing4 = bq.fetchMissingSidecars(broot2, fuluBlock2, + peerCustodyColumns2) + + check: + compareSidecars( + broot1, + sidecars1.toOpenArray(i, len(sidecars1) - 1), missing1) == true + compareSidecars( + broot2, + sidecars2.toOpenArray(i, len(sidecars2) - 1), missing2) == true + + check: + compareIdentifiers(expected1[i], missing3) + len(missing4) == 0 + + if i >= len(sidecars1): + break + + bq.put(broot1, sidecars1[i]) + bq.put(broot2, sidecars2[i]) + + bq.remove(broot1) + bq.remove(broot2) + check len(bq) == 0 + + test "put()/fetchMissingSidecars/remove test [supernode]": + let + custodyColumns = supernodeColumns() + peerCustodyColumns1 = + [63, 64, 65, 66, 95, 96, 97, 98].mapIt(ColumnIndex(it)) + + var bq = ColumnQuarantine.init(cfg, custodyColumns, nil) + let + broot1 = genBlockRoot(1) + broot2 = genBlockRoot(2) + sidecars1 = + block: + var res: seq[ref DataColumnSidecar] + for i in 0 ..< (len(custodyColumns) div 2 + 1): + res.add(newClone(genDataColumnSidecar( + index = int(custodyColumns[i]), slot = 1, proposer_index = 5))) + res + sidecars2 = + block: + var res: seq[ref DataColumnSidecar] + for i in 0 ..< (len(custodyColumns) div 2 + 1): + res.add(newClone(genDataColumnSidecar( + index = int(custodyColumns[i]), slot = 2, proposer_index = 50))) + res + commitments1 = [ + genKzgCommitment(1), genKzgCommitment(2), genKzgCommitment(3) + ] + commitments2 = [ + genKzgCommitment(4), genKzgCommitment(5), genKzgCommitment(6) + ] + fuluBlock1 = genFuluSignedBeaconBlock(broot1, commitments1) + fuluBlock2 = genFuluSignedBeaconBlock(broot2, commitments2) + + func checkSupernodeExpected( + root: Eth2Digest, + index: int, + missing: openArray[DataColumnIdentifier] + ): bool = + const ExpectedVectors = [ + (@[63, 64, 65, 66, 95, 96, 97, 98], 0 .. 57), + (@[63, 64, 65, 66, 95, 96, 97], 58 .. 58), + (@[63, 64, 65, 66, 95, 96], 59 .. 59), + (@[63, 64, 65, 66, 95], 60 .. 60), + (@[63, 64, 65, 66], 61 .. 61), + (@[63, 64, 65], 62 .. 62), + (@[63, 64], 63 .. 63), + (@[64], 64 .. 64), + (@[], 65 .. 65) + ] + + doAssert(index in 0 .. 65) + for expect in ExpectedVectors: + if index in expect[1]: + if len(expect[0]) != len(missing): + return false + for i in 0 ..< len(missing): + if (missing[i].block_root != root) or + (int(missing[i].index) != expect[0][i]): + return false + return true + false + + for i in 0 ..< len(sidecars1) + 1: + let + missing1 = bq.fetchMissingSidecars(broot1, fuluBlock1) + missing2 = bq.fetchMissingSidecars(broot2, fuluBlock2) + missing3 = bq.fetchMissingSidecars(broot1, fuluBlock1, + peerCustodyColumns1) + check: + compareSidecars( + broot1, + sidecars1.toOpenArray(i, len(sidecars1) - 1), missing1) == true + compareSidecars( + broot2, + sidecars2.toOpenArray(i, len(sidecars2) - 1), missing2) == true + checkSupernodeExpected( + broot1, + i, missing3) == true + + if i >= len(sidecars1): + break + + bq.put(broot1, sidecars1[i]) + bq.put(broot2, sidecars2[i]) + + bq.remove(broot1) + bq.remove(broot2) + check len(bq) == 0 + + test "popSidecars()/hasSidecars() return []/true on block without columns": + let + custodyColumns = + [63, 64, 65, 66, 95, 96, 97, 98].mapIt(ColumnIndex(it)) + var + bq = ColumnQuarantine.init(cfg, custodyColumns, nil) + let + blockRoot1 = genBlockRoot(100) + blockRoot2 = genBlockRoot(5337) + blockRoot3 = genBlockRoot(1294967295) + fuluBlock1 = genFuluSignedBeaconBlock(blockRoot1, []) + fuluBlock2 = genFuluSignedBeaconBlock(blockRoot2, []) + fuluBlock3 = genFuluSignedBeaconBlock(blockRoot3, []) + + check: + bq.hasSidecars(fuluBlock1.root, fuluBlock1) == true + bq.hasSidecars(fuluBlock2.root, fuluBlock2) == true + bq.hasSidecars(fuluBlock3.root, fuluBlock3) == true + + let + res1 = bq.popSidecars(fuluBlock1.root, fuluBlock1) + res2 = bq.popSidecars(fuluBlock2.root, fuluBlock2) + res3 = bq.popSidecars(fuluBlock3.root, fuluBlock3) + + check: + res1.isOk() + len(res1.get()) == 0 + res2.isOk() + len(res2.get()) == 0 + res3.isOk() + len(res3.get()) == 0 + + test "overfill protection test": + let + custodyColumns = + [63, 64, 65, 66, 95, 96, 97, 98].mapIt(ColumnIndex(it)) + + var + bq = ColumnQuarantine.init(cfg, custodyColumns, nil) + sidecars: seq[tuple[sidecar: ref DataColumnSidecar, + blockRoot: Eth2Digest]] + + let maxSidecars = int(NUMBER_OF_COLUMNS * SLOTS_PER_EPOCH) * 3 + for i in 0 ..< maxSidecars: + let + index = i mod len(custodyColumns) + slot = i div len(custodyColumns) + 100 + blockRoot = genBlockRoot(slot) + sidecar = newClone( + genDataColumnSidecar(index = int(custodyColumns[index]), + slot, proposer_index = i)) + sidecars.add((sidecar, blockRoot)) + + for item in sidecars: + bq.put(item.blockRoot, item.sidecar) + + check len(bq) == maxSidecars + + # put(sidecar) test + + for i in 0 ..< len(custodyColumns): + check: + bq.hasSidecar( + blockRoot = + genBlockRoot( + int(sidecars[i].sidecar[].signed_block_header.message.slot)), + slot = + sidecars[i].sidecar[].signed_block_header.message.slot, + proposer_index = + sidecars[i].sidecar[].signed_block_header.message.proposer_index, + index = sidecars[i].sidecar[].index + ) == true + + let + sidecar = newClone( + genDataColumnSidecar(index = int(custodyColumns[0]), + slot = 10000, proposer_index = 1000000)) + blockRoot = genBlockRoot(10000) + check: + bq.hasSidecar(blockRoot = blockRoot, slot = Slot(10000), + proposer_index = 1000000'u64, + index = custodyColumns[0]) == false + bq.put(blockRoot, sidecar) + check: + len(bq) == (len(sidecars) - len(custodyColumns) + 1) + bq.hasSidecar(blockRoot = blockRoot, slot = Slot(10000), + proposer_index = 1000000'u64, + index = custodyColumns[0]) == true + + for i in 0 ..< len(custodyColumns): + check: + bq.hasSidecar( + blockRoot = + genBlockRoot( + int(sidecars[i].sidecar[].signed_block_header.message.slot)), + slot = + sidecars[i].sidecar[].signed_block_header.message.slot, + proposer_index = + sidecars[i].sidecar[].signed_block_header.message.proposer_index, + index = sidecars[i].sidecar[].index + ) == false + + # put(openArray[sidecar]) test + + let + msidecars = + block: + var res: seq[ref DataColumnSidecar] + for i in 0 ..< len(custodyColumns): + let sidecar = + newClone(genDataColumnSidecar(index = int(custodyColumns[i]), + slot = 100_000, + proposer_index = 2000000)) + res.add(sidecar) + res + mblockRoot = genBlockRoot(20000) + + check: + len(bq) == (len(sidecars) - len(custodyColumns) + 1) + + let beforeLength = len(bq) + + for s in msidecars: + check: + bq.hasSidecar(mblockRoot, + s.signed_block_header.message.slot, + s.signed_block_header.message.proposer_index, + s.index) == false + + bq.put(mblockRoot, msidecars) + check len(bq) == beforeLength + + for s in msidecars: + check: + bq.hasSidecar(mblockRoot, + s.signed_block_header.message.slot, + s.signed_block_header.message.proposer_index, + s.index) == true + + for i in 0 ..< len(custodyColumns): + let j = len(custodyColumns) + i + check: + bq.hasSidecar( + blockRoot = + genBlockRoot( + int(sidecars[j].sidecar[].signed_block_header.message.slot)), + slot = + sidecars[j].sidecar[].signed_block_header.message.slot, + proposer_index = + sidecars[j].sidecar[].signed_block_header.message.proposer_index, + index = sidecars[j].sidecar[].index + ) == false + + test "put() duplicate items should not affect counters": + let + custodyColumns = + [63, 64, 65, 66, 95, 96, 97, 98].mapIt(ColumnIndex(it)) + var + bq = ColumnQuarantine.init(cfg, custodyColumns, nil) + sidecars1: seq[ref DataColumnSidecar] + sidecars1d: seq[ref DataColumnSidecar] + sidecars2: seq[ref DataColumnSidecar] + sidecars2d: seq[ref DataColumnSidecar] + + for index in custodyColumns: + let + sidecar1 = newClone(genDataColumnSidecar(int(index), 1, 64)) + sidecar1d = newClone(genDataColumnSidecar(int(index), 1, 64)) + sidecar2 = newClone(genDataColumnSidecar(int(index), 2, 65)) + sidecar2d = newClone(genDataColumnSidecar(int(index), 2, 65)) + sidecars1.add(sidecar1) + sidecars1d.add(sidecar1d) + sidecars2.add(sidecar2) + sidecars2d.add(sidecar2d) + + let + broot1 = genBlockRoot(100) + broot2 = genBlockRoot(200) + fuluBlock1 = genFuluSignedBeaconBlock(broot1, [genKzgCommitment(1)]) + fuluBlock2 = genFuluSignedBeaconBlock(broot2, [genKzgCommitment(2)]) + + check: + len(bq) == 0 + len(bq.fetchMissingSidecars(broot1, fuluBlock1, custodyColumns)) == + len(custodyColumns) + len(bq.fetchMissingSidecars(broot2, fuluBlock2, custodyColumns)) == + len(custodyColumns) + + for index in 0 ..< len(custodyColumns): + bq.put(broot1, sidecars1[index]) + check: + len(bq) == (index + 1) + len(bq.fetchMissingSidecars(broot1, fuluBlock1, custodyColumns)) == + len(custodyColumns) - (index + 1) + bq.put(broot1, sidecars1d[index]) + check: + len(bq) == (index + 1) + len(bq.fetchMissingSidecars(broot1, fuluBlock1, custodyColumns)) == + len(custodyColumns) - (index + 1) + + for index in 0 ..< len(custodyColumns): + bq.put(broot2, sidecars2[index]) + check: + len(bq) == len(custodyColumns) + (index + 1) + len(bq.fetchMissingSidecars(broot2, fuluBlock2, custodyColumns)) == + len(custodyColumns) - (index + 1) + bq.put(broot2, sidecars2d[index]) + check: + len(bq) == len(custodyColumns) + (index + 1) + len(bq.fetchMissingSidecars(broot2, fuluBlock2, custodyColumns)) == + len(custodyColumns) - (index + 1) + + bq.remove(broot2) + check len(bq) == len(custodyColumns) + bq.remove(broot1) + check len(bq) == 0 + + test "pruneAfterFinalization() test": + let + custodyColumns = + [63, 64, 65, 66, 95, 96, 97, 98].mapIt(ColumnIndex(it)) + + const TestVectors = [ + (root: 1, slot: 1, index: 63, proposer_index: 20), + (root: 1, slot: 1, index: 64, proposer_index: 20), + (root: 1, slot: 1, index: 65, proposer_index: 20), + (root: 1, slot: 1, index: 66, proposer_index: 20), + (root: 1, slot: 1, index: 96, proposer_index: 20), + (root: 2, slot: 32, index: 63, proposer_index: 21), + (root: 2, slot: 32, index: 64, proposer_index: 21), + (root: 2, slot: 32, index: 65, proposer_index: 21), + (root: 3, slot: 33, index: 63, proposer_index: 22), + (root: 3, slot: 33, index: 64, proposer_index: 22), + (root: 4, slot: 63, index: 63, proposer_index: 23), + (root: 5, slot: 64, index: 63, proposer_index: 24), + (root: 5, slot: 64, index: 64, proposer_index: 24), + (root: 5, slot: 64, index: 65, proposer_index: 24), + (root: 6, slot: 65, index: 63, proposer_index: 25), + (root: 6, slot: 65, index: 64, proposer_index: 25), + (root: 7, slot: 67, index: 63, proposer_index: 26), + (root: 7, slot: 67, index: 64, proposer_index: 26), + (root: 8, slot: 95, index: 63, proposer_index: 27), + (root: 8, slot: 95, index: 64, proposer_index: 27), + (root: 8, slot: 95, index: 65, proposer_index: 27), + (root: 8, slot: 95, index: 66, proposer_index: 27), + (root: 8, slot: 95, index: 98, proposer_index: 27), + (root: 9, slot: 96, index: 63, proposer_index: 28), + (root: 9, slot: 96, index: 64, proposer_index: 28), + (root: 9, slot: 96, index: 65, proposer_index: 28), + (root: 9, slot: 96, index: 66, proposer_index: 28), + (root: 9, slot: 96, index: 95, proposer_index: 28), + (root: 9, slot: 96, index: 96, proposer_index: 28), + (root: 9, slot: 96, index: 97, proposer_index: 28), + (root: 9, slot: 96, index: 98, proposer_index: 28), + (root: 10, slot: 127, index: 96, proposer_index: 29), + (root: 10, slot: 127, index: 97, proposer_index: 29), + (root: 10, slot: 127, index: 98, proposer_index: 29) + ] + + var bq = ColumnQuarantine.init(cfg, custodyColumns, nil) + for item in TestVectors: + let sidecar = + newClone( + genDataColumnSidecar(index = item.index, slot = item.slot, + proposer_index = item.proposer_index)) + bq.put(genBlockRoot(item.root), sidecar) + + check: + len(bq) == len(TestVectors) + + for item in TestVectors: + check: + bq.hasSidecar( + genBlockRoot(item.root), Slot(item.slot), + uint64(item.proposer_index), BlobIndex(item.index)) == true + + bq.pruneAfterFinalization(Epoch(1)) + check: + len(bq) == len(TestVectors) - 5 + + for item in TestVectors: + let res = + if item.root == 1: + false + else: + true + check: + bq.hasSidecar( + genBlockRoot(item.root), Slot(item.slot), + uint64(item.proposer_index), BlobIndex(item.index)) == res + + bq.pruneAfterFinalization(Epoch(2)) + check: + len(bq) == len(TestVectors) - 5 - 6 + + for item in TestVectors: + let res = + if item.root in [1, 2, 3, 4]: + false + else: + true + check: + bq.hasSidecar( + genBlockRoot(item.root), Slot(item.slot), + uint64(item.proposer_index), BlobIndex(item.index)) == res + + bq.pruneAfterFinalization(Epoch(3)) + check: + len(bq) == len(TestVectors) - 5 - 6 - 12 + + for item in TestVectors: + let res = + if item.root in [1, 2, 3, 4, 5, 6, 7, 8]: + false + else: + true + check: + bq.hasSidecar( + genBlockRoot(item.root), Slot(item.slot), + uint64(item.proposer_index), BlobIndex(item.index)) == res + + bq.pruneAfterFinalization(Epoch(4)) + check: + len(bq) == 0 + + for item in TestVectors: + check: + bq.hasSidecar( + genBlockRoot(item.root), Slot(item.slot), + uint64(item.proposer_index), BlobIndex(item.index)) == false