ncli_db: add putState, putBlock

These tools allow modifying an existing nimbus database for the purpose
of recovery or reorg, moving the head, tail and genesis to arbitrary
points.

* remove potentially expensive `putState` in `BeaconStateDB`
* introduce `latest_block_root` which computes the root of the latest
applied block from the `latest_block_header` field (instead of passing
it in separately)
* avoid some unnecessary BeaconState copies during init
* discover nim-lang/Nim#19094
* prefer `HashedBeaconState` in a few places to avoid recomputing state
root
* fetch latest block root from state when creating blocks
* harden `get_beacon_proposer_index` against invalid slots and document
* move random spec function tests to `test_spec.nim`
* avoid unnecessary state root computation before block proposal

.gitignore

@@ -23,6 +23,7 @@ build/
 *.la
 *.exe
 *.dll
+*.su
 
 /scripts/testnet*.sh
 

AllTests-mainnet.md

@@ -67,8 +67,10 @@ OK: 16/16 Fail: 0/16 Skip: 0/16
 ## Beacon state [Preset: mainnet]
 ```diff
 + Smoke test initialize_beacon_state_from_eth1 [Preset: mainnet]                             OK
++ get_beacon_proposer_index                                                                  OK
++ latest_block_root                                                                          OK
 ```
-OK: 1/1 Fail: 0/1 Skip: 0/1
+OK: 3/3 Fail: 0/3 Skip: 0/3
 ## Block pool processing [Preset: mainnet]
 ```diff
 + Adding the same block twice returns a Duplicate error [Preset: mainnet]                    OK
@@ -364,4 +366,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 OK: 1/1 Fail: 0/1 Skip: 0/1
 
 ---TOTAL---
-OK: 204/206 Fail: 0/206 Skip: 2/206
+OK: 206/208 Fail: 0/208 Skip: 2/208

beacon_chain/beacon_chain_db.nim

@@ -535,9 +535,6 @@ proc putState*(db: BeaconChainDB, key: Eth2Digest, value: merge.BeaconState) =
   db.mergeStatesNoVal.putSnappySSZ(
     key.data, toBeaconStateNoImmutableValidators(value))
 
-proc putState*(db: BeaconChainDB, value: ForkyBeaconState) =
-  db.putState(hash_tree_root(value), value)
-
 # For testing rollback
 proc putCorruptPhase0State*(db: BeaconChainDB, key: Eth2Digest) =
   db.statesNoVal.putSnappySSZ(key.data, Validator())

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -667,9 +667,9 @@ proc putState(dag: ChainDAGRef, state: StateData) =
   # Ideally we would save the state and the root lookup cache in a single
   # transaction to prevent database inconsistencies, but the state loading code
   # is resilient against one or the other going missing
-  withState(state.data): dag.db.putState(state.root, state.data)
-  dag.db.putStateRoot(
-    state.blck.root, getStateField(state.data, slot), getStateRoot(state.data))
+  withState(state.data):
+    dag.db.putStateRoot(state.latest_block_root(), state.data.slot, state.root)
+    dag.db.putState(state.root, state.data)
 
   debug "Stored state", putStateDur = Moment.now() - startTick
 
@@ -1394,12 +1394,12 @@ proc preInit*(
           tail_genesis_validators_root = shortLog(tail_genesis_validators_root)
         quit 1
 
-      let blck = get_initial_beacon_block(state.data)
+      let blck = get_initial_beacon_block(state)
       db.putGenesisBlock(blck.root)
       db.putBlock(blck)
 
+      db.putStateRoot(state.latest_block_root(), state.data.slot, state.root)
       db.putState(state.root, state.data)
-      db.putStateRoot(blck.root, state.data.slot, state.root)
       blck.root
     else: # tail and genesis are the same
       withBlck(tailBlock):
@@ -1422,8 +1422,8 @@ proc preInit*(
       db.putTailBlock(blck.root)
       db.putHeadBlock(blck.root)
 
+      db.putStateRoot(state.latest_block_root(), state.data.slot, state.root)
       db.putState(state.root, state.data)
-      db.putStateRoot(blck.root, state.data.slot, state.root)
 
       notice "New database from snapshot",
         genesisBlockRoot = shortLog(genesisBlockRoot),

beacon_chain/eth1/eth1_monitor.nim

@@ -208,11 +208,11 @@ when hasGenesisDetection:
 
     var deposits = m.allGenesisDepositsUpTo(eth1Block.voteData.deposit_count)
 
-    result = initialize_beacon_state_from_eth1(
+    result = newClone(initialize_beacon_state_from_eth1(
       m.cfg,
       eth1Block.voteData.block_hash,
       eth1Block.timestamp.uint64,
-      deposits, {})
+      deposits, {}))
 
     if eth1Block.activeValidatorsCount != 0:
       doAssert result.validators.lenu64 == eth1Block.activeValidatorsCount

beacon_chain/nimbus_beacon_node.nim

@@ -177,7 +177,7 @@ proc init*(T: type BeaconNode,
 
     if config.finalizedCheckpointBlock.isNone:
       withState(checkpointState[]):
-        if getStateField(checkpointState[], slot) > 0:
+        if state.data.slot > 0:
           fatal "Specifying a non-genesis --finalized-checkpoint-state requires specifying --finalized-checkpoint-block as well"
           quit 1
     else:
@@ -1530,8 +1530,8 @@ proc doCreateTestnet(config: BeaconNodeConf, rng: var BrHmacDrbgContext) {.raise
                else: (waitFor getEth1BlockHash(config.web3Urls[0], blockId("latest"))).asEth2Digest
     cfg = getRuntimeConfig(config.eth2Network)
   var
-    initialState = initialize_beacon_state_from_eth1(
-      cfg, eth1Hash, startTime, deposits, {skipBlsValidation})
+    initialState = newClone(initialize_beacon_state_from_eth1(
+      cfg, eth1Hash, startTime, deposits, {skipBlsValidation}))
 
   # https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#create-genesis-state
   initialState.genesis_time = startTime

beacon_chain/spec/beaconstate.nim

@@ -195,7 +195,7 @@ proc initialize_beacon_state_from_eth1*(
     eth1_block_hash: Eth2Digest,
     eth1_timestamp: uint64,
     deposits: openArray[DepositData],
-    flags: UpdateFlags = {}): phase0.BeaconStateRef {.nbench.} =
+    flags: UpdateFlags = {}): phase0.BeaconState {.nbench.} =
   ## Get the genesis ``BeaconState``.
   ##
   ## Before the beacon chain starts, validators will register in the Eth1 chain
@@ -212,7 +212,9 @@ proc initialize_beacon_state_from_eth1*(
   # at that point :)
   doAssert deposits.lenu64 >= SLOTS_PER_EPOCH
 
-  var state = phase0.BeaconStateRef(
+  # TODO https://github.com/nim-lang/Nim/issues/19094
+  template state(): untyped = result
+  state = phase0.BeaconState(
     fork: genesisFork(cfg),
     genesis_time: genesis_time_from_eth1_timestamp(cfg, eth1_timestamp),
     eth1_data:
@@ -243,7 +245,7 @@ proc initialize_beacon_state_from_eth1*(
 
     pubkeyToIndex.withValue(pubkey, foundIdx) do:
       # Increase balance by deposit amount
-      increase_balance(state[], ValidatorIndex foundIdx[], amount)
+      increase_balance(state, ValidatorIndex foundIdx[], amount)
     do:
       if skipBlsValidation in flags or
          verify_deposit_signature(cfg, deposit):
@@ -274,50 +276,52 @@ proc initialize_beacon_state_from_eth1*(
   # Set genesis validators root for domain separation and chain versioning
   state.genesis_validators_root = hash_tree_root(state.validators)
 
-  state
+  # TODO https://github.com/nim-lang/Nim/issues/19094
+  # state
 
 proc initialize_hashed_beacon_state_from_eth1*(
     cfg: RuntimeConfig,
     eth1_block_hash: Eth2Digest,
     eth1_timestamp: uint64,
     deposits: openArray[DepositData],
     flags: UpdateFlags = {}): phase0.HashedBeaconState =
-  let genesisState = initialize_beacon_state_from_eth1(
-    cfg, eth1_block_hash, eth1_timestamp, deposits, flags)
-  phase0.HashedBeaconState(
-    data: genesisState[], root: hash_tree_root(genesisState[]))
+  # TODO https://github.com/nim-lang/Nim/issues/19094
+  result = phase0.HashedBeaconState(
+    data: initialize_beacon_state_from_eth1(
+      cfg, eth1_block_hash, eth1_timestamp, deposits, flags))
+  result.root = hash_tree_root(result.data)
 
 # https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/phase0/beacon-chain.md#genesis-block
-func get_initial_beacon_block*(state: phase0.BeaconState):
+func get_initial_beacon_block*(state: phase0.HashedBeaconState):
     phase0.TrustedSignedBeaconBlock =
   # The genesis block is implicitly trusted
   let message = phase0.TrustedBeaconBlock(
-    slot: state.slot,
-    state_root: hash_tree_root(state),)
+    slot: state.data.slot,
+    state_root: state.root)
     # parent_root, randao_reveal, eth1_data, signature, and body automatically
     # initialized to default values.
   phase0.TrustedSignedBeaconBlock(
     message: message, root: hash_tree_root(message))
 
 # https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/altair/beacon-chain.md#initialize-state-for-pure-altair-testnets-and-test-vectors
-func get_initial_beacon_block*(state: altair.BeaconState):
+func get_initial_beacon_block*(state: altair.HashedBeaconState):
     altair.TrustedSignedBeaconBlock =
   # The genesis block is implicitly trusted
   let message = altair.TrustedBeaconBlock(
-    slot: state.slot,
-    state_root: hash_tree_root(state),)
+    slot: state.data.slot,
+    state_root: state.root)
     # parent_root, randao_reveal, eth1_data, signature, and body automatically
     # initialized to default values.
   altair.TrustedSignedBeaconBlock(
     message: message, root: hash_tree_root(message))
 
 # https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/merge/beacon-chain.md#testing
-func get_initial_beacon_block*(state: merge.BeaconState):
+func get_initial_beacon_block*(state: merge.HashedBeaconState):
     merge.TrustedSignedBeaconBlock =
   # The genesis block is implicitly trusted
   let message = merge.TrustedBeaconBlock(
-    slot: state.slot,
-    state_root: hash_tree_root(state),)
+    slot: state.data.slot,
+    state_root: state.root,)
     # parent_root, randao_reveal, eth1_data, signature, and body automatically
     # initialized to default values.
   merge.TrustedSignedBeaconBlock(
@@ -326,7 +330,7 @@ func get_initial_beacon_block*(state: merge.BeaconState):
 func get_initial_beacon_block*(state: ForkedHashedBeaconState):
     ForkedTrustedSignedBeaconBlock =
   withState(state):
-    ForkedTrustedSignedBeaconBlock.init(get_initial_beacon_block(state.data))
+    ForkedTrustedSignedBeaconBlock.init(get_initial_beacon_block(state))
 
 # https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/phase0/beacon-chain.md#get_block_root_at_slot
 func get_block_root_at_slot*(state: ForkyBeaconState, slot: Slot): Eth2Digest =
@@ -894,3 +898,27 @@ func upgrade_to_merge*(cfg: RuntimeConfig, pre: altair.BeaconState):
 template isValidInState*(idx: ValidatorIndex, state: ForkyBeaconState): bool =
   idx.uint64 < state.validators.lenu64
 
+func latest_block_root*(state: ForkyBeaconState, state_root: Eth2Digest): Eth2Digest =
+  # The root of the last block that was successfully applied to this state -
+  # normally, when a block is applied, the data from the header is stored in
+  # the state without the state root - on the next process_slot, the state root
+  # is added to the header and the block root can now be computed and added to
+  # the block roots table. If process_slot has not yet run on top of the new
+  # block, we must fill in the state root ourselves.
+  if state.slot == state.latest_block_header.slot:
+    # process_slot will not yet have updated the header of the "current" block -
+    # similar to block creation, we fill it in with the state root
+    var tmp = state.latest_block_header
+    tmp.state_root = state_root
+    hash_tree_root(tmp)
+  elif state.slot <=
+      (state.latest_block_header.slot + SLOTS_PER_HISTORICAL_ROOT):
+    # block_roots is limited to about a day - see assert in
+    # `get_block_root_at_slot`
+    state.get_block_root_at_slot(state.latest_block_header.slot)
+  else:
+    # Reallly long periods of empty slots - unlikely but possible
+    hash_tree_root(state.latest_block_header)
+
+func latest_block_root*(state: ForkyHashedBeaconState): Eth2Digest =
+  latest_block_root(state.data, state.root)