Merge branch 'unstable' into kurtosis-check

beacon_chain/gossip_processing/batch_validation.nim

@@ -99,7 +99,7 @@ type
   Batch* = object
     ## A batch represents up to BatchedCryptoSize non-aggregated signatures
     created: Moment
-    sigsets: seq[SignatureSet]
+    multiSets: Table[array[32, byte], MultiSignatureSet]
     items: seq[BatchItem]
 
   VerifierItem = object
@@ -198,7 +198,7 @@ proc complete(batchCrypto: var BatchCrypto, batch: var Batch, ok: bool) =
 
   batchCrypto.counts.batches += 1
   batchCrypto.counts.signatures += batch.items.len()
-  batchCrypto.counts.aggregates += batch.sigsets.len()
+  batchCrypto.counts.aggregates += batch.multiSets.len()
 
   if batchCrypto.counts.batches >= 256:
     # Not too often, so as not to overwhelm our metrics
@@ -227,12 +227,29 @@ proc spawnBatchVerifyTask(tp: Taskpool, task: ptr BatchTask) =
   # Possibly related to: https://github.com/nim-lang/Nim/issues/22305
   tp.spawn batchVerifyTask(task)
 
-proc batchVerifyAsync*(
-    verifier: ref BatchVerifier, signal: ThreadSignalPtr,
+func combine(
+    multiSet: MultiSignatureSet,
+    verifier: ref BatchVerifier): SignatureSet =
+  var secureRandomBytes: array[32, byte]
+  verifier[].rng[].generate(secureRandomBytes)
+  multiSet.combine(secureRandomBytes)
+
+func combineAll(
+    multiSets: Table[array[32, byte], MultiSignatureSet],
+    verifier: ref BatchVerifier): seq[SignatureSet] =
+  var sigsets = newSeqOfCap[SignatureSet](multiSets.len)
+  for multiSet in multiSets.values():
+    sigsets.add multiSet.combine(verifier)
+  sigsets
+
+proc batchVerifyAsync(
+    verifier: ref BatchVerifier,
+    signal: ThreadSignalPtr,
     batch: ref Batch): Future[bool] {.async: (raises: [CancelledError]).} =
+  let sigsets = batch[].multiSets.combineAll(verifier)
   var task = BatchTask(
-    setsPtr: makeUncheckedArray(baseAddr batch[].sigsets),
-    numSets: batch[].sigsets.len,
+    setsPtr: makeUncheckedArray(baseAddr sigsets),
+    numSets: sigsets.len,
     taskpool: verifier[].taskpool,
     cache: addr verifier[].sigVerifCache,
     signal: signal,
@@ -254,18 +271,18 @@ proc batchVerifyAsync*(
   task.ok.load()
 
 proc processBatch(
-    batchCrypto: ref BatchCrypto, batch: ref Batch,
-    verifier: ref BatchVerifier, signal: ThreadSignalPtr) {.async: (raises: [CancelledError]).} =
-  let
-    numSets = batch[].sigsets.len()
+    batchCrypto: ref BatchCrypto,
+    batch: ref Batch,
+    verifier: ref BatchVerifier,
+    signal: ThreadSignalPtr) {.async: (raises: [CancelledError]).} =
+  let numSets = batch[].multiSets.len
 
   if numSets == 0:
     # Nothing to do in this batch, can happen when a batch is created without
     # there being any signatures successfully added to it
     return
 
-  let
-    startTick = Moment.now()
+  let startTick = Moment.now()
 
   # If the hardware is too slow to keep up or an event caused a temporary
   # buildup of signature verification tasks, the batch will be dropped so as to
@@ -290,13 +307,19 @@ proc processBatch(
     # may not be beneficial to use batch verification:
     # https://github.com/status-im/nim-blscurve/blob/3956f63dd0ed5d7939f6195ee09e4c5c1ace9001/blscurve/bls_batch_verifier.nim#L390
     if numSets == 1:
-      blsVerify(batch[].sigsets[0])
+      var r: bool
+      for multiSet in batch[].multiSets.values():
+        r = blsVerify(multiSet.combine(verifier))
+        break
+      r
     elif batchCrypto[].taskpool.numThreads > 1 and numSets > 3:
       await batchVerifyAsync(verifier, signal, batch)
     else:
       let secureRandomBytes = verifier[].rng[].generate(array[32, byte])
       batchVerifySerial(
-        verifier[].sigVerifCache, batch.sigsets, secureRandomBytes)
+        verifier[].sigVerifCache,
+        batch.multiSets.combineAll(verifier),
+        secureRandomBytes)
 
   trace "batch crypto - finished",
     numSets, items = batch[].items.len(), ok,
@@ -356,11 +379,10 @@ proc verifySoon(
     batch = batchCrypto[].getBatch()
     fut = newFuture[BatchResult](name)
 
-  # TODO If there is a signature set `item in batch[].sigsets.mitems()`
-  # with `item.message == sigset.message`, further performance could be gained
-  # by implementing Pippenger multi-scalar multiplication in `nim-blscurve`.
-  # https://gist.github.com/wemeetagain/d52fc4b077f80db6e423935244c2afb2
-  batch[].sigsets.add sigset
+  batch[].multiSets.withValue(sigset.message, multiSet):
+    multiSet[].add sigset
+  do:
+    batch[].multiSets[sigset.message] = MultiSignatureSet.init sigset
 
   # We need to keep the "original" sigset to allow verifying each signature
   # one by one in the case the combined operation fails

beacon_chain/gossip_processing/block_processor.nim

@@ -13,7 +13,7 @@ import
   ../sszdump
 
 from std/deques import Deque, addLast, contains, initDeque, items, len, shrink
-from std/sequtils import mapIt
+from std/sequtils import anyIt, mapIt
 from ../consensus_object_pools/consensus_manager import
   ConsensusManager, checkNextProposer, optimisticExecutionBlockHash,
   runProposalForkchoiceUpdated, shouldSyncOptimistically, updateHead,
@@ -541,31 +541,32 @@ proc storeBlock(
 
   if NewPayloadStatus.noResponse == payloadStatus:
     # When the execution layer is not available to verify the payload, we do the
-    # required check on the CL side instead and proceed as if the EL was syncing
-
-    # TODO run https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/beacon-chain.md#blob-kzg-commitments
-    # https://github.com/ethereum/execution-apis/blob/main/src/engine/experimental/blob-extension.md#specification
-    # "This validation MUST be instantly run in all cases even during active
-    # sync process."
-    #
-    # Client software MUST validate `blockHash` value as being equivalent to
-    # `Keccak256(RLP(ExecutionBlockHeader))`
-    # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.3/src/engine/paris.md#specification
-    #
-    # This should simulate an unsynced EL, which still must perform these
-    # checks. This means it must be able to do so without context, beyond
-    # whatever data the block itself contains.
+    # required checks on the CL instead and proceed as if the EL was syncing
+    # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.4/specs/bellatrix/beacon-chain.md#verify_and_notify_new_payload
+    # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.4/specs/deneb/beacon-chain.md#modified-verify_and_notify_new_payload
     when typeof(signedBlock).kind >= ConsensusFork.Bellatrix:
-      template payload(): auto = signedBlock.message.body.execution_payload
-      if  signedBlock.message.is_execution_block and
-          payload.block_hash !=
+      if signedBlock.message.is_execution_block:
+        template payload(): auto = signedBlock.message.body.execution_payload
+
+        template returnWithError(msg: string): untyped =
+          debug msg, executionPayload = shortLog(payload)
+          self[].dumpInvalidBlock(signedBlock)
+          doAssert strictVerification notin dag.updateFlags
+          self.consensusManager.quarantine[].addUnviable(signedBlock.root)
+          return err((VerifierError.Invalid, ProcessingStatus.completed))
+
+        if payload.transactions.anyIt(it.len == 0):
+          returnWithError "Execution block contains zero length transactions"
+
+        if payload.block_hash !=
             signedBlock.message.compute_execution_block_hash():
-        debug "Execution block hash validation failed",
-          execution_payload = shortLog(payload)
-        self[].dumpInvalidBlock(signedBlock)
-        doAssert strictVerification notin dag.updateFlags
-        self.consensusManager.quarantine[].addUnviable(signedBlock.root)
-        return err((VerifierError.Invalid, ProcessingStatus.completed))
+          returnWithError "Execution block hash validation failed"
+
+        # [New in Deneb:EIP4844]
+        # TODO run https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/beacon-chain.md#blob-kzg-commitments
+        # https://github.com/ethereum/execution-apis/blob/main/src/engine/experimental/blob-extension.md#specification
+        # "This validation MUST be instantly run in all cases even during active
+        # sync process."
 
   let newPayloadTick = Moment.now()
 

beacon_chain/spec/crypto.nim

@@ -245,14 +245,14 @@ proc blsVerify*(
   # Guard against invalid signature blobs that fail to parse
   parsedSig.isSome() and blsVerify(pubkey, message, parsedSig.get())
 
-func blsVerify*(sigSet: SignatureSet): bool =
+func blsVerify*(sigset: SignatureSet): bool =
   ## Unbatched verification
   ## of 1 SignatureSet
   ## tuple[pubkey: blscurve.PublicKey, message: array[32, byte], blscurve.signature: Signature]
   verify(
-    sigSet.pubkey,
-    sigSet.message,
-    sigSet.signature
+    sigset.pubkey,
+    sigset.message,
+    sigset.signature
   )
 
 func blsSign*(privkey: ValidatorPrivKey, message: openArray[byte]): CookedSig =