Skip to content

Commit

Permalink
ncli_db: add putState, putBlock
Browse files Browse the repository at this point in the history
These tools allow modifying an existing nimbus database for the purpose
of recovery or reorg, moving the head, tail and genesis to arbitrary
points.

* remove potentially expensive `putState` in `BeaconStateDB`
* introduce `latest_block_root` which computes the root of the latest
applied block from the `latest_block_header` field (instead of passing
it in separately)
* avoid some unnecessary BeaconState copies during init
* discover nim-lang/Nim#19094
* prefer `HashedBeaconState` in a few places to avoid recomputing state
root
* fetch latest block root from state when creating blocks
* harden `get_beacon_proposer_index` against invalid slots and document
* move random spec function tests to `test_spec.nim`
* avoid unnecessary state root computation before block proposal
  • Loading branch information
arnetheduck committed Nov 15, 2021
1 parent 222674b commit b5f6561
Show file tree
Hide file tree
Showing 27 changed files with 384 additions and 255 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ build/
*.la
*.exe
*.dll
*.su

/scripts/testnet*.sh

Expand Down
6 changes: 4 additions & 2 deletions AllTests-mainnet.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,10 @@ OK: 16/16 Fail: 0/16 Skip: 0/16
## Beacon state [Preset: mainnet]
```diff
+ Smoke test initialize_beacon_state_from_eth1 [Preset: mainnet] OK
+ get_beacon_proposer_index OK
+ latest_block_root OK
```
OK: 1/1 Fail: 0/1 Skip: 0/1
OK: 3/3 Fail: 0/3 Skip: 0/3
## Block pool processing [Preset: mainnet]
```diff
+ Adding the same block twice returns a Duplicate error [Preset: mainnet] OK
Expand Down Expand Up @@ -364,4 +366,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
OK: 1/1 Fail: 0/1 Skip: 0/1

---TOTAL---
OK: 204/206 Fail: 0/206 Skip: 2/206
OK: 206/208 Fail: 0/208 Skip: 2/208
3 changes: 0 additions & 3 deletions beacon_chain/beacon_chain_db.nim
Original file line number Diff line number Diff line change
Expand Up @@ -535,9 +535,6 @@ proc putState*(db: BeaconChainDB, key: Eth2Digest, value: merge.BeaconState) =
db.mergeStatesNoVal.putSnappySSZ(
key.data, toBeaconStateNoImmutableValidators(value))

proc putState*(db: BeaconChainDB, value: ForkyBeaconState) =
db.putState(hash_tree_root(value), value)

# For testing rollback
proc putCorruptPhase0State*(db: BeaconChainDB, key: Eth2Digest) =
db.statesNoVal.putSnappySSZ(key.data, Validator())
Expand Down
12 changes: 6 additions & 6 deletions beacon_chain/consensus_object_pools/blockchain_dag.nim
Original file line number Diff line number Diff line change
Expand Up @@ -667,9 +667,9 @@ proc putState(dag: ChainDAGRef, state: StateData) =
# Ideally we would save the state and the root lookup cache in a single
# transaction to prevent database inconsistencies, but the state loading code
# is resilient against one or the other going missing
withState(state.data): dag.db.putState(state.root, state.data)
dag.db.putStateRoot(
state.blck.root, getStateField(state.data, slot), getStateRoot(state.data))
withState(state.data):
dag.db.putStateRoot(state.latest_block_root(), state.data.slot, state.root)
dag.db.putState(state.root, state.data)

debug "Stored state", putStateDur = Moment.now() - startTick

Expand Down Expand Up @@ -1394,12 +1394,12 @@ proc preInit*(
tail_genesis_validators_root = shortLog(tail_genesis_validators_root)
quit 1

let blck = get_initial_beacon_block(state.data)
let blck = get_initial_beacon_block(state)
db.putGenesisBlock(blck.root)
db.putBlock(blck)

db.putStateRoot(state.latest_block_root(), state.data.slot, state.root)
db.putState(state.root, state.data)
db.putStateRoot(blck.root, state.data.slot, state.root)
blck.root
else: # tail and genesis are the same
withBlck(tailBlock):
Expand All @@ -1422,8 +1422,8 @@ proc preInit*(
db.putTailBlock(blck.root)
db.putHeadBlock(blck.root)

db.putStateRoot(state.latest_block_root(), state.data.slot, state.root)
db.putState(state.root, state.data)
db.putStateRoot(blck.root, state.data.slot, state.root)

notice "New database from snapshot",
genesisBlockRoot = shortLog(genesisBlockRoot),
Expand Down
4 changes: 2 additions & 2 deletions beacon_chain/eth1/eth1_monitor.nim
Original file line number Diff line number Diff line change
Expand Up @@ -208,11 +208,11 @@ when hasGenesisDetection:

var deposits = m.allGenesisDepositsUpTo(eth1Block.voteData.deposit_count)

result = initialize_beacon_state_from_eth1(
result = newClone(initialize_beacon_state_from_eth1(
m.cfg,
eth1Block.voteData.block_hash,
eth1Block.timestamp.uint64,
deposits, {})
deposits, {}))

if eth1Block.activeValidatorsCount != 0:
doAssert result.validators.lenu64 == eth1Block.activeValidatorsCount
Expand Down
6 changes: 3 additions & 3 deletions beacon_chain/nimbus_beacon_node.nim
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ proc init*(T: type BeaconNode,

if config.finalizedCheckpointBlock.isNone:
withState(checkpointState[]):
if getStateField(checkpointState[], slot) > 0:
if state.data.slot > 0:
fatal "Specifying a non-genesis --finalized-checkpoint-state requires specifying --finalized-checkpoint-block as well"
quit 1
else:
Expand Down Expand Up @@ -1530,8 +1530,8 @@ proc doCreateTestnet(config: BeaconNodeConf, rng: var BrHmacDrbgContext) {.raise
else: (waitFor getEth1BlockHash(config.web3Urls[0], blockId("latest"))).asEth2Digest
cfg = getRuntimeConfig(config.eth2Network)
var
initialState = initialize_beacon_state_from_eth1(
cfg, eth1Hash, startTime, deposits, {skipBlsValidation})
initialState = newClone(initialize_beacon_state_from_eth1(
cfg, eth1Hash, startTime, deposits, {skipBlsValidation}))

# https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#create-genesis-state
initialState.genesis_time = startTime
Expand Down
64 changes: 46 additions & 18 deletions beacon_chain/spec/beaconstate.nim
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ proc initialize_beacon_state_from_eth1*(
eth1_block_hash: Eth2Digest,
eth1_timestamp: uint64,
deposits: openArray[DepositData],
flags: UpdateFlags = {}): phase0.BeaconStateRef {.nbench.} =
flags: UpdateFlags = {}): phase0.BeaconState {.nbench.} =
## Get the genesis ``BeaconState``.
##
## Before the beacon chain starts, validators will register in the Eth1 chain
Expand All @@ -212,7 +212,9 @@ proc initialize_beacon_state_from_eth1*(
# at that point :)
doAssert deposits.lenu64 >= SLOTS_PER_EPOCH

var state = phase0.BeaconStateRef(
# TODO https://github.com/nim-lang/Nim/issues/19094
template state(): untyped = result
state = phase0.BeaconState(
fork: genesisFork(cfg),
genesis_time: genesis_time_from_eth1_timestamp(cfg, eth1_timestamp),
eth1_data:
Expand Down Expand Up @@ -243,7 +245,7 @@ proc initialize_beacon_state_from_eth1*(

pubkeyToIndex.withValue(pubkey, foundIdx) do:
# Increase balance by deposit amount
increase_balance(state[], ValidatorIndex foundIdx[], amount)
increase_balance(state, ValidatorIndex foundIdx[], amount)
do:
if skipBlsValidation in flags or
verify_deposit_signature(cfg, deposit):
Expand Down Expand Up @@ -274,50 +276,52 @@ proc initialize_beacon_state_from_eth1*(
# Set genesis validators root for domain separation and chain versioning
state.genesis_validators_root = hash_tree_root(state.validators)

state
# TODO https://github.com/nim-lang/Nim/issues/19094
# state

proc initialize_hashed_beacon_state_from_eth1*(
cfg: RuntimeConfig,
eth1_block_hash: Eth2Digest,
eth1_timestamp: uint64,
deposits: openArray[DepositData],
flags: UpdateFlags = {}): phase0.HashedBeaconState =
let genesisState = initialize_beacon_state_from_eth1(
cfg, eth1_block_hash, eth1_timestamp, deposits, flags)
phase0.HashedBeaconState(
data: genesisState[], root: hash_tree_root(genesisState[]))
# TODO https://github.com/nim-lang/Nim/issues/19094
result = phase0.HashedBeaconState(
data: initialize_beacon_state_from_eth1(
cfg, eth1_block_hash, eth1_timestamp, deposits, flags))
result.root = hash_tree_root(result.data)

# https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/phase0/beacon-chain.md#genesis-block
func get_initial_beacon_block*(state: phase0.BeaconState):
func get_initial_beacon_block*(state: phase0.HashedBeaconState):
phase0.TrustedSignedBeaconBlock =
# The genesis block is implicitly trusted
let message = phase0.TrustedBeaconBlock(
slot: state.slot,
state_root: hash_tree_root(state),)
slot: state.data.slot,
state_root: state.root)
# parent_root, randao_reveal, eth1_data, signature, and body automatically
# initialized to default values.
phase0.TrustedSignedBeaconBlock(
message: message, root: hash_tree_root(message))

# https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/altair/beacon-chain.md#initialize-state-for-pure-altair-testnets-and-test-vectors
func get_initial_beacon_block*(state: altair.BeaconState):
func get_initial_beacon_block*(state: altair.HashedBeaconState):
altair.TrustedSignedBeaconBlock =
# The genesis block is implicitly trusted
let message = altair.TrustedBeaconBlock(
slot: state.slot,
state_root: hash_tree_root(state),)
slot: state.data.slot,
state_root: state.root)
# parent_root, randao_reveal, eth1_data, signature, and body automatically
# initialized to default values.
altair.TrustedSignedBeaconBlock(
message: message, root: hash_tree_root(message))

# https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/merge/beacon-chain.md#testing
func get_initial_beacon_block*(state: merge.BeaconState):
func get_initial_beacon_block*(state: merge.HashedBeaconState):
merge.TrustedSignedBeaconBlock =
# The genesis block is implicitly trusted
let message = merge.TrustedBeaconBlock(
slot: state.slot,
state_root: hash_tree_root(state),)
slot: state.data.slot,
state_root: state.root,)
# parent_root, randao_reveal, eth1_data, signature, and body automatically
# initialized to default values.
merge.TrustedSignedBeaconBlock(
Expand All @@ -326,7 +330,7 @@ func get_initial_beacon_block*(state: merge.BeaconState):
func get_initial_beacon_block*(state: ForkedHashedBeaconState):
ForkedTrustedSignedBeaconBlock =
withState(state):
ForkedTrustedSignedBeaconBlock.init(get_initial_beacon_block(state.data))
ForkedTrustedSignedBeaconBlock.init(get_initial_beacon_block(state))

# https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/phase0/beacon-chain.md#get_block_root_at_slot
func get_block_root_at_slot*(state: ForkyBeaconState, slot: Slot): Eth2Digest =
Expand Down Expand Up @@ -894,3 +898,27 @@ func upgrade_to_merge*(cfg: RuntimeConfig, pre: altair.BeaconState):
template isValidInState*(idx: ValidatorIndex, state: ForkyBeaconState): bool =
idx.uint64 < state.validators.lenu64

func latest_block_root*(state: ForkyBeaconState, state_root: Eth2Digest): Eth2Digest =
# The root of the last block that was successfully applied to this state -
# normally, when a block is applied, the data from the header is stored in
# the state without the state root - on the next process_slot, the state root
# is added to the header and the block root can now be computed and added to
# the block roots table. If process_slot has not yet run on top of the new
# block, we must fill in the state root ourselves.
if state.slot == state.latest_block_header.slot:
# process_slot will not yet have updated the header of the "current" block -
# similar to block creation, we fill it in with the state root
var tmp = state.latest_block_header
tmp.state_root = state_root
hash_tree_root(tmp)
elif state.slot <=
(state.latest_block_header.slot + SLOTS_PER_HISTORICAL_ROOT):
# block_roots is limited to about a day - see assert in
# `get_block_root_at_slot`
state.get_block_root_at_slot(state.latest_block_header.slot)
else:
# Reallly long periods of empty slots - unlikely but possible
hash_tree_root(state.latest_block_header)

func latest_block_root*(state: ForkyHashedBeaconState): Eth2Digest =
latest_block_root(state.data, state.root)
Loading

0 comments on commit b5f6561

Please sign in to comment.