SNO-285 setup local beacon testnet for e2e tests (paritytech#661)

* Starts with setting up CL in E2E tests

* Progress on beacon network

* Attempts to get script to work

* Simple example for debugging

* Finally got the lodestar dependencies to import.

* Update script with latest `next` code.

* Fix script.

* Progress on private beacon net

* Adds jwt token

* Lodestar setup script

* Updated dependencies

* Try the local setup again.

* Beacon node local testnet function added

* Cleanup

* Fixes

* Config for local net

* Adds constants

* Cleans up errors so main problem is evident

* Pallet config constant progress

* Try something else

* Swap constants based on config.

* Cleanup

* Revert to usize for bitvector.

* Revert changes.

* Revert unnecessary changes.

* Local beacon net testing fixes

* Temp config update to test minimal config

* Testing minimal spec

* Update API endpoints and use public Lodestar Ropsten server for start-services.

* Remove echo.

* Adds config replacement for beacon endpoint

* Finishing off local beacon testnet

* Last bit of cleanup

* Final tweaks

* PR comments

* Reverts config

Co-authored-by: claravanstaden <Cats 4 life!>

.gitignore

@@ -6,3 +6,6 @@ workspace.code-workspace
 
 # Node modules
 node_modules/
+
+#Intellij project file
+.idea

parachain/pallets/ethereum-beacon-client/Cargo.toml

@@ -62,3 +62,4 @@ runtime-benchmarks = [
     "frame-support/runtime-benchmarks",
     "frame-system/runtime-benchmarks",
 ]
+minimal = []

parachain/pallets/ethereum-beacon-client/src/config.rs

@@ -1,17 +1,39 @@
-pub const MAX_PROPOSER_SLASHINGS: usize = 16;
+#[cfg(feature = "minimal")]
+mod minimal;
 
-pub const MAX_ATTESTER_SLASHINGS: usize =  2;
+#[cfg(not(feature = "minimal"))]
+mod mainnet;
 
-pub const MAX_ATTESTATIONS: usize =  128;
+#[cfg(feature = "minimal")]
+pub use minimal::*;
 
-pub const MAX_DEPOSITS: usize =  16;
+#[cfg(not(feature = "minimal"))]
+pub use mainnet::*;
 
-pub const MAX_VOLUNTARY_EXITS: usize =  16;
+use snowbridge_beacon_primitives::ForkVersion;
+
+pub const CURRENT_SYNC_COMMITTEE_INDEX: u64 = 22;
+pub const CURRENT_SYNC_COMMITTEE_DEPTH: u64 = 5;
+
+pub const NEXT_SYNC_COMMITTEE_DEPTH: u64 = 5;
+pub const NEXT_SYNC_COMMITTEE_INDEX: u64 = 23;
+
+pub const FINALIZED_ROOT_DEPTH: u64 = 6;
+pub const FINALIZED_ROOT_INDEX: u64 = 41;
 
+pub const MAX_PROPOSER_SLASHINGS: usize = 16;
+pub const MAX_ATTESTER_SLASHINGS: usize =  2;
+pub const MAX_ATTESTATIONS: usize =  128;
+pub const MAX_DEPOSITS: usize =  16;
+pub const MAX_VOLUNTARY_EXITS: usize =  16;
 pub const MAX_VALIDATORS_PER_COMMITTEE: usize = 2048;
+pub const MAX_EXTRA_DATA_BYTES: usize = 32;
 
 pub const DEPOSIT_CONTRACT_TREE_DEPTH: usize = 32;
 
-pub const MAX_EXTRA_DATA_BYTES: usize = 32;
+/// GENESIS_FORK_VERSION('0x00000000')
+pub const GENESIS_FORK_VERSION: ForkVersion = [30, 30, 30, 30];
 
-pub const SYNC_COMMITTEE_SIZE: usize = 512;
+/// DomainType('0x07000000')
+/// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#domain-types
+pub const DOMAIN_SYNC_COMMITTEE: [u8; 4] = [7, 0, 0, 0];

parachain/pallets/ethereum-beacon-client/src/config/mainnet.rs

@@ -0,0 +1,3 @@
+pub const SLOTS_PER_EPOCH: u64 = 32;
+pub const EPOCHS_PER_SYNC_COMMITTEE_PERIOD: u64 = 256;
+pub const SYNC_COMMITTEE_SIZE: usize = 512;

parachain/pallets/ethereum-beacon-client/src/config/minimal.rs

@@ -0,0 +1,3 @@
+pub const SLOTS_PER_EPOCH: u64 = 8;
+pub const EPOCHS_PER_SYNC_COMMITTEE_PERIOD: u64 = 8;
+pub const SYNC_COMMITTEE_SIZE: usize = 32;

parachain/pallets/ethereum-beacon-client/src/lib.rs

@@ -2,12 +2,12 @@
 #![cfg_attr(not(feature = "std"), no_std)]
 
 mod merkleization;
+pub mod config;
 #[cfg(test)]
 mod mock;
 #[cfg(test)]
 mod tests;
 mod ssz;
-mod config;
 
 use codec::{Decode, Encode};
 use frame_support::{dispatch::DispatchResult, log, transactional};
@@ -17,31 +17,9 @@ use sp_core::H256;
 use sp_io::hashing::sha2_256;
 use sp_runtime::RuntimeDebug;
 use sp_std::prelude::*;
-use snowbridge_beacon_primitives::{SyncCommittee, BeaconHeader, SyncAggregate, ForkData, Root, Domain, PublicKey, SigningData, ExecutionHeader, BeaconBlock};
+use snowbridge_beacon_primitives::{SyncCommittee, BeaconHeader, SyncAggregate, ForkData, Root, Domain, PublicKey, SigningData, ExecutionHeader, BeaconBlock, ProofBranch, ForkVersion};
 use snowbridge_core::{Message, Verifier};
-
-const SLOTS_PER_EPOCH: u64 = 32;
-
-const EPOCHS_PER_SYNC_COMMITTEE_PERIOD: u64 = 256;
-
-const CURRENT_SYNC_COMMITTEE_INDEX: u64 = 22;
-const CURRENT_SYNC_COMMITTEE_DEPTH: u64 = 5;
-
-const NEXT_SYNC_COMMITTEE_DEPTH: u64 = 5;
-const NEXT_SYNC_COMMITTEE_INDEX: u64 = 23;
-
-const FINALIZED_ROOT_DEPTH: u64 = 6;
-const FINALIZED_ROOT_INDEX: u64 = 41;
-
-/// GENESIS_FORK_VERSION('0x00000000')
-const GENESIS_FORK_VERSION: ForkVersion = [30, 30, 30, 30];
-
-/// DomainType('0x07000000')
-/// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#domain-types
-const DOMAIN_SYNC_COMMITTEE: [u8; 4] = [7, 0, 0, 0];
-
-type ProofBranch = Vec<H256>;
-type ForkVersion = [u8; 4];
+use crate::merkleization::get_sync_committee_bits;
 
 #[derive(Clone, Default, Encode, Decode, PartialEq, RuntimeDebug, TypeInfo)]
 #[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))]
@@ -174,14 +152,21 @@ pub mod pallet {
 	#[pallet::genesis_build]
 	impl<T: Config> GenesisBuild<T> for GenesisConfig {
 		fn build(&self) {
+			log::info!(
+				target: "ethereum-beacon-client",
+				"💫 Sync committee size is: {}",
+				config::SYNC_COMMITTEE_SIZE
+			);
+
 			Pallet::<T>::initial_sync(
 				self.initial_sync.clone(),
 			).unwrap();
 		}
 	}
 
 	#[pallet::call]
-	impl<T: Config> Pallet<T> {
+	impl<T: Config> Pallet<T>
+	{
 		#[pallet::weight(1_000_000)]
 		#[transactional]
 		pub fn sync_committee_period_update(
@@ -292,8 +277,8 @@ pub mod pallet {
 				initial_sync.current_sync_committee.clone(),
 				initial_sync.current_sync_committee_branch,
 				initial_sync.header.state_root,
-				CURRENT_SYNC_COMMITTEE_DEPTH,
-				CURRENT_SYNC_COMMITTEE_INDEX,
+				config::CURRENT_SYNC_COMMITTEE_DEPTH,
+				config::CURRENT_SYNC_COMMITTEE_INDEX,
 			)?;
 
 			let period = Self::compute_current_sync_period(initial_sync.header.slot);
@@ -311,15 +296,15 @@ pub mod pallet {
 		fn process_sync_committee_period_update(
 			update: SyncCommitteePeriodUpdate,
 		) -> DispatchResult {
-			let sync_committee_bits = merkleization::get_sync_committee_bits(update.sync_aggregate.sync_committee_bits.clone())
+			let sync_committee_bits = get_sync_committee_bits(update.sync_aggregate.sync_committee_bits.clone())
 				.map_err(|_| DispatchError::Other("Couldn't process sync committee bits"))?;
 			Self::sync_committee_participation_is_supermajority(sync_committee_bits.clone())?;
 			Self::verify_sync_committee(
 				update.next_sync_committee.clone(),
 				update.next_sync_committee_branch,
 				update.finalized_header.state_root,
-				NEXT_SYNC_COMMITTEE_DEPTH,
-				NEXT_SYNC_COMMITTEE_INDEX,
+				config::NEXT_SYNC_COMMITTEE_DEPTH,
+				config::NEXT_SYNC_COMMITTEE_INDEX,
 			)?;
 
 			let block_root: H256 = merkleization::hash_tree_root_beacon_header(update.finalized_header.clone())
@@ -328,8 +313,8 @@ pub mod pallet {
 				block_root,
 				update.finality_branch,
 				update.attested_header.state_root,
-				FINALIZED_ROOT_DEPTH,
-				FINALIZED_ROOT_INDEX,
+				config::FINALIZED_ROOT_DEPTH,
+				config::FINALIZED_ROOT_INDEX,
 			)?;
 
 			let current_period = Self::compute_current_sync_period(update.attested_header.slot);
@@ -352,7 +337,7 @@ pub mod pallet {
 		}
 
 		fn process_finalized_header(update: FinalizedHeaderUpdate) -> DispatchResult {
-			let sync_committee_bits = merkleization::get_sync_committee_bits(update.sync_aggregate.sync_committee_bits.clone())
+			let sync_committee_bits = get_sync_committee_bits(update.sync_aggregate.sync_committee_bits.clone())
 				.map_err(|_| DispatchError::Other("Couldn't process sync committee bits"))?;
 			Self::sync_committee_participation_is_supermajority(sync_committee_bits.clone())?;
 
@@ -362,8 +347,8 @@ pub mod pallet {
 				block_root,
 				update.finality_branch,
 				update.attested_header.state_root,
-				FINALIZED_ROOT_DEPTH,
-				FINALIZED_ROOT_INDEX,
+				config::FINALIZED_ROOT_DEPTH,
+				config::FINALIZED_ROOT_INDEX,
 			)?;
 
 			let current_period = Self::compute_current_sync_period(update.attested_header.slot);
@@ -414,7 +399,7 @@ pub mod pallet {
 			};
 
 			let validators_root = <ValidatorsRoot<T>>::get();
-			let sync_committee_bits = merkleization::get_sync_committee_bits(update.sync_aggregate.sync_committee_bits.clone())
+			let sync_committee_bits = get_sync_committee_bits(update.sync_aggregate.sync_committee_bits.clone())
 				.map_err(|_| DispatchError::Other("Couldn't process sync committee bits"))?;
 			Self::verify_signed_header(
 				sync_committee_bits,
@@ -476,7 +461,7 @@ pub mod pallet {
 				}
 			}
 
-			let domain_type = DOMAIN_SYNC_COMMITTEE.to_vec();
+			let domain_type = config::DOMAIN_SYNC_COMMITTEE.to_vec();
 			// Domains are used for for seeds, for signatures, and for selecting aggregators.
 			let domain = Self::compute_domain(domain_type, Some(fork_version), validators_root)?;
 			// Hash tree root of SigningData - object root + domain
@@ -599,6 +584,12 @@ pub mod pallet {
 
 			let latest_committee_period = <LatestSyncCommitteePeriod<T>>::get();
 
+			log::trace!(
+				target: "ethereum-beacon-client",
+				"💫 Saved sync committee for period {}.",
+				period
+			);
+
 			if period > latest_committee_period {
 				log::trace!(
 					target: "ethereum-beacon-client",
@@ -652,7 +643,7 @@ pub mod pallet {
 		}
 
 		pub(super) fn compute_current_sync_period(slot: u64) -> u64 {
-			slot / SLOTS_PER_EPOCH / EPOCHS_PER_SYNC_COMMITTEE_PERIOD
+			slot / config::SLOTS_PER_EPOCH / config::EPOCHS_PER_SYNC_COMMITTEE_PERIOD
 		}
 
 		/// Return the domain for the domain_type and fork_version.
@@ -663,7 +654,7 @@ pub mod pallet {
 		) -> Result<Domain, DispatchError> {
 			let unwrapped_fork_version: ForkVersion;
 			if fork_version.is_none() {
-				unwrapped_fork_version = GENESIS_FORK_VERSION;
+				unwrapped_fork_version = config::GENESIS_FORK_VERSION;
 			} else {
 				unwrapped_fork_version = fork_version.unwrap();
 			}

parachain/pallets/ethereum-beacon-client/src/merkleization.rs

@@ -7,7 +7,7 @@ use ssz_rs::U256;
 use byte_slice_cast::AsByteSlice;
 use snowbridge_beacon_primitives::{SyncAggregate, Attestation, Checkpoint, Eth1Data, BeaconHeader, AttesterSlashing, ExecutionPayload, SigningData, ForkData, SyncCommittee, AttestationData, Body, ProposerSlashing, Deposit, VoluntaryExit};
 use crate::ssz::*;
-use crate::config;
+use crate::config as config;
 
 #[derive(Debug)]
 pub enum MerkleizationError {
@@ -227,7 +227,7 @@ pub fn hash_tree_root_sync_committee(sync_committee: SyncCommittee) -> Result<[u
         pubkeys_vec.push(conv_pubkey);
     }
 
-    let pubkeys = Vector::<Vector::<u8, 48>, 512>::from_iter(pubkeys_vec.clone());
+    let pubkeys = Vector::<Vector::<u8, 48>, { config::SYNC_COMMITTEE_SIZE }>::from_iter(pubkeys_vec.clone());
 
     let agg = Vector::<u8, 48>::from_iter(sync_committee.aggregate_pubkey.0);
 

parachain/pallets/ethereum-beacon-client/src/ssz.rs

@@ -2,7 +2,7 @@ use ssz_rs_derive::SimpleSerialize;
 use ssz_rs::{Deserialize, Sized, Bitlist, Bitvector, U256};
 use ssz_rs::prelude::{Vector, List};
 use sp_std::{vec::Vec, vec};
-use crate::config;
+use crate::config as config;
 
 #[derive(Default, Debug, SimpleSerialize, Clone)]
 pub struct SSZVoluntaryExit {
@@ -98,13 +98,13 @@ pub struct SSZBeaconBlockHeader {
 
 #[derive(Default, SimpleSerialize)]
 pub struct SSZSyncCommittee {
-    pub pubkeys: Vector<Vector<u8, 48>, 512>,
+    pub pubkeys: Vector<Vector<u8, 48>, { config::SYNC_COMMITTEE_SIZE }>,
     pub aggregate_pubkey: Vector<u8, 48>,
 }
 
 #[derive(Default, Debug, SimpleSerialize, Clone)]
 pub struct SSZSyncAggregate {
-    pub sync_committee_bits: Bitvector<{ config::SYNC_COMMITTEE_SIZE} >,
+    pub sync_committee_bits: Bitvector<{ config::SYNC_COMMITTEE_SIZE }>,
     pub sync_committee_signature: Vector<u8, 96>,
 }
 

parachain/primitives/beacon/src/lib.rs

@@ -18,6 +18,9 @@ use core::fmt::Formatter;
 pub type Root = H256;
 pub type Domain = H256;
 pub type ValidatorIndex = u64;
+pub type ProofBranch = Vec<H256>;
+pub type ForkVersion = [u8; 4];
+
 
 #[derive(Clone, Encode, Decode, PartialEq, RuntimeDebug, TypeInfo)]
 pub struct PublicKey(pub [u8; 48]);

parachain/runtime/snowbase/Cargo.toml

@@ -76,7 +76,7 @@ basic-channel = { path = "../../pallets/basic-channel", package = "snowbridge-ba
 incentivized-channel = { path = "../../pallets/incentivized-channel", package = "snowbridge-incentivized-channel", default-features = false }
 dispatch = { path = "../../pallets/dispatch", package = "snowbridge-dispatch", default-features = false }
 ethereum-light-client = { path = "../../pallets/ethereum-light-client", package = "snowbridge-ethereum-light-client", default-features = false }
-ethereum-beacon-client = { path = "../../pallets/ethereum-beacon-client", package = "snowbridge-ethereum-beacon-client", default-features = false }
+ethereum-beacon-client = { path = "../../pallets/ethereum-beacon-client", package = "snowbridge-ethereum-beacon-client", default-features = false, features=["minimal"]}
 dot-app = { path = "../../pallets/dot-app", package = "snowbridge-dot-app", default-features = false }
 eth-app = { path = "../../pallets/eth-app", package = "snowbridge-eth-app", default-features = false }
 erc20-app = { path = "../../pallets/erc20-app", package = "snowbridge-erc20-app", default-features = false }

parachain/runtime/snowblink/Cargo.toml

@@ -75,7 +75,7 @@ basic-channel = { path = "../../pallets/basic-channel", package = "snowbridge-ba
 incentivized-channel = { path = "../../pallets/incentivized-channel", package = "snowbridge-incentivized-channel", default-features = false }
 dispatch = { path = "../../pallets/dispatch", package = "snowbridge-dispatch", default-features = false }
 ethereum-light-client = { path = "../../pallets/ethereum-light-client", package = "snowbridge-ethereum-light-client", default-features = false }
-ethereum-beacon-client = { path = "../../pallets/ethereum-beacon-client", package = "snowbridge-ethereum-beacon-client", default-features = false }
+ethereum-beacon-client = { path = "../../pallets/ethereum-beacon-client", package = "snowbridge-ethereum-beacon-client", default-features = false}
 dot-app = { path = "../../pallets/dot-app", package = "snowbridge-dot-app", default-features = false }
 eth-app = { path = "../../pallets/eth-app", package = "snowbridge-eth-app", default-features = false }
 erc20-app = { path = "../../pallets/erc20-app", package = "snowbridge-erc20-app", default-features = false }

parachain/runtime/snowbridge/Cargo.toml

@@ -75,7 +75,7 @@ basic-channel = { path = "../../pallets/basic-channel", package = "snowbridge-ba
 incentivized-channel = { path = "../../pallets/incentivized-channel", package = "snowbridge-incentivized-channel", default-features = false }
 dispatch = { path = "../../pallets/dispatch", package = "snowbridge-dispatch", default-features = false }
 ethereum-light-client = { path = "../../pallets/ethereum-light-client", package = "snowbridge-ethereum-light-client", default-features = false }
-ethereum-beacon-client = { path = "../../pallets/ethereum-beacon-client", package = "snowbridge-ethereum-beacon-client", default-features = false }
+ethereum-beacon-client = { path = "../../pallets/ethereum-beacon-client", package = "snowbridge-ethereum-beacon-client", default-features = false}
 dot-app = { path = "../../pallets/dot-app", package = "snowbridge-dot-app", default-features = false }
 eth-app = { path = "../../pallets/eth-app", package = "snowbridge-eth-app", default-features = false }
 erc20-app = { path = "../../pallets/erc20-app", package = "snowbridge-erc20-app", default-features = false }

relayer/relays/beacon/config.go

@@ -7,6 +7,16 @@ type Config struct {
 	Sink   SinkConfig   `mapstructure:"sink"`
 }
 
+type SpecSettings struct {
+	SlotsInEpoch                 uint64 `mapstructure:"slotsInEpoch"`
+	EpochsPerSyncCommitteePeriod uint64 `mapstructure:"epochsPerSyncCommitteePeriod"`
+}
+
+type Spec struct {
+	Minimal SpecSettings `mapstructure:"minimal"`
+	Mainnet SpecSettings `mapstructure:"mainnet"`
+}
+
 type SourceConfig struct {
 	Beacon    BeaconConfig          `mapstructure:"beacon"`
 	Ethereum  config.EthereumConfig `mapstructure:"ethereum"`
@@ -21,8 +31,18 @@ type ContractsConfig struct {
 type BeaconConfig struct {
 	Endpoint                string `mapstructure:"endpoint"`
 	FinalizedUpdateEndpoint string `mapstructure:"finalizedUpdateEndpoint"`
+	Spec                    Spec   `mapstructure:"spec"`
+	ActiveSpec              string `mapstructure:"activeSpec"`
 }
 
 type SinkConfig struct {
 	Parachain config.ParachainConfig `mapstructure:"parachain"`
 }
+
+func (c Config) GetSpecSettings() SpecSettings {
+	if c.Source.Beacon.ActiveSpec == "minimal" {
+		return c.Source.Beacon.Spec.Minimal
+	}
+
+	return c.Source.Beacon.Spec.Mainnet
+}