Merge latest unstable to das branch

Cargo.toml

@@ -9,6 +9,7 @@ members = [
     "beacon_node/client",
     "beacon_node/eth1",
     "beacon_node/lighthouse_network",
+    "beacon_node/lighthouse_network/gossipsub",
     "beacon_node/execution_layer",
     "beacon_node/http_api",
     "beacon_node/http_metrics",
@@ -200,6 +201,7 @@ execution_layer = { path = "beacon_node/execution_layer" }
 filesystem = { path = "common/filesystem" }
 fork_choice = { path = "consensus/fork_choice" }
 genesis = { path = "beacon_node/genesis" }
+gossipsub = { path = "beacon_node/lighthouse_network/gossipsub/" }
 http_api = { path = "beacon_node/http_api" }
 int_to_bytes = { path = "consensus/int_to_bytes" }
 kzg = { path = "crypto/kzg" }

Makefile

@@ -39,7 +39,7 @@ PROFILE ?= release
 
 # List of all hard forks. This list is used to set env variables for several tests so that
 # they run for different forks.
-FORKS=phase0 altair merge capella deneb
+FORKS=phase0 altair merge capella deneb electra
 
 # Extra flags for Cargo
 CARGO_INSTALL_EXTRA_FLAGS?=

README.md

@@ -41,7 +41,7 @@ as the canonical staking deposit contract address.
 The [Lighthouse Book](https://lighthouse-book.sigmaprime.io) contains information for users and
 developers.
 
-The Lighthouse team maintains a blog at [lighthouse-blog.sigmaprime.io][blog] which contains periodical
+The Lighthouse team maintains a blog at [lighthouse-blog.sigmaprime.io][blog] which contains periodic
 progress updates, roadmap insights and interesting findings.
 
 ## Branches

account_manager/README.md

@@ -29,6 +29,6 @@ Simply run `./account_manager generate` to generate a new random private key,
 which will be automatically saved to the correct directory.
 
 If you prefer to use our "deterministic" keys for testing purposes, simply
-run `./accounts_manager generate_deterministic -i <index>`, where `index` is
+run `./account_manager generate_deterministic -i <index>`, where `index` is
 the validator index for the key. This will reliably produce the same key each time
-and save it to the directory.
+and save it to the directory.

account_manager/src/lib.rs

@@ -22,7 +22,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
 }
 
 /// Run the account manager, returning an error if the operation did not succeed.
-pub fn run<T: EthSpec>(matches: &ArgMatches<'_>, env: Environment<T>) -> Result<(), String> {
+pub fn run<E: EthSpec>(matches: &ArgMatches<'_>, env: Environment<E>) -> Result<(), String> {
     match matches.subcommand() {
         (wallet::CMD, Some(matches)) => wallet::cli_run(matches)?,
         (validator::CMD, Some(matches)) => validator::cli_run(matches, env)?,

account_manager/src/validator/create.rs

@@ -112,9 +112,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         )
 }
 
-pub fn cli_run<T: EthSpec>(
+pub fn cli_run<E: EthSpec>(
     matches: &ArgMatches,
-    env: Environment<T>,
+    env: Environment<E>,
     validator_dir: PathBuf,
 ) -> Result<(), String> {
     let spec = env.core_context().eth2_config.spec;

account_manager/src/validator/mod.rs

@@ -39,7 +39,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         .subcommand(exit::cli_app())
 }
 
-pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<(), String> {
+pub fn cli_run<E: EthSpec>(matches: &ArgMatches, env: Environment<E>) -> Result<(), String> {
     let validator_base_dir = if matches.value_of("datadir").is_some() {
         let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
         path.join(DEFAULT_VALIDATOR_DIR)
@@ -49,7 +49,7 @@ pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<
     eprintln!("validator-dir path: {:?}", validator_base_dir);
 
     match matches.subcommand() {
-        (create::CMD, Some(matches)) => create::cli_run::<T>(matches, env, validator_base_dir),
+        (create::CMD, Some(matches)) => create::cli_run::<E>(matches, env, validator_base_dir),
         (modify::CMD, Some(matches)) => modify::cli_run(matches, validator_base_dir),
         (import::CMD, Some(matches)) => import::cli_run(matches, validator_base_dir),
         (list::CMD, Some(_)) => list::cli_run(validator_base_dir),

account_manager/src/validator/slashing_protection.rs

@@ -53,9 +53,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         )
 }
 
-pub fn cli_run<T: EthSpec>(
+pub fn cli_run<E: EthSpec>(
     matches: &ArgMatches<'_>,
-    env: Environment<T>,
+    env: Environment<E>,
     validator_base_dir: PathBuf,
 ) -> Result<(), String> {
     let slashing_protection_db_path = validator_base_dir.join(SLASHING_PROTECTION_FILENAME);
@@ -64,7 +64,7 @@ pub fn cli_run<T: EthSpec>(
         .ok_or("Unable to get testnet configuration from the environment")?;
 
     let genesis_validators_root = eth2_network_config
-        .genesis_validators_root::<T>()?
+        .genesis_validators_root::<E>()?
         .ok_or_else(|| "Unable to get genesis state, has genesis occurred?".to_string())?;
 
     match matches.subcommand() {

beacon_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "beacon_node"
-version = "5.1.1"
+version = "5.1.3"
 authors = [
     "Paul Hauner <[email protected]>",
     "Age Manning <[email protected]",

beacon_node/beacon_chain/src/attestation_rewards.rs

@@ -1,28 +1,14 @@
 use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
 use eth2::lighthouse::attestation_rewards::{IdealAttestationRewards, TotalAttestationRewards};
 use eth2::lighthouse::StandardAttestationRewards;
-use participation_cache::ParticipationCache;
+use eth2::types::ValidatorId;
 use safe_arith::SafeArith;
 use serde_utils::quoted_u64::Quoted;
 use slog::debug;
+use state_processing::common::base::{self, SqrtTotalActiveBalance};
 use state_processing::per_epoch_processing::altair::{
-    process_inactivity_updates, process_justification_and_finalization,
-};
-use state_processing::{
-    common::altair::BaseRewardPerIncrement,
-    per_epoch_processing::altair::{participation_cache, rewards_and_penalties::get_flag_weight},
-};
-use std::collections::HashMap;
-use store::consts::altair::{
-    PARTICIPATION_FLAG_WEIGHTS, TIMELY_HEAD_FLAG_INDEX, TIMELY_SOURCE_FLAG_INDEX,
-    TIMELY_TARGET_FLAG_INDEX,
+    process_inactivity_updates_slow, process_justification_and_finalization,
 };
-use types::consts::altair::WEIGHT_DENOMINATOR;
-
-use types::{BeaconState, Epoch, EthSpec};
-
-use eth2::types::ValidatorId;
-use state_processing::common::base::get_base_reward_from_effective_balance;
 use state_processing::per_epoch_processing::base::rewards_and_penalties::{
     get_attestation_component_delta, get_attestation_deltas_all, get_attestation_deltas_subset,
     get_inactivity_penalty_delta, get_inclusion_delay_delta,
@@ -32,6 +18,19 @@ use state_processing::per_epoch_processing::base::{
     process_justification_and_finalization as process_justification_and_finalization_base,
     TotalBalances, ValidatorStatus, ValidatorStatuses,
 };
+use state_processing::{
+    common::altair::BaseRewardPerIncrement,
+    common::update_progressive_balances_cache::initialize_progressive_balances_cache,
+    epoch_cache::initialize_epoch_cache,
+    per_epoch_processing::altair::rewards_and_penalties::get_flag_weight,
+};
+use std::collections::HashMap;
+use store::consts::altair::{
+    PARTICIPATION_FLAG_WEIGHTS, TIMELY_HEAD_FLAG_INDEX, TIMELY_SOURCE_FLAG_INDEX,
+    TIMELY_TARGET_FLAG_INDEX,
+};
+use types::consts::altair::WEIGHT_DENOMINATOR;
+use types::{BeaconState, Epoch, EthSpec, RelativeEpoch};
 
 impl<T: BeaconChainTypes> BeaconChain<T> {
     pub fn compute_attestation_rewards(
@@ -57,7 +56,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             BeaconState::Altair(_)
             | BeaconState::Merge(_)
             | BeaconState::Capella(_)
-            | BeaconState::Deneb(_) => self.compute_attestation_rewards_altair(state, validators),
+            | BeaconState::Deneb(_)
+            | BeaconState::Electra(_) => self.compute_attestation_rewards_altair(state, validators),
         }
     }
 
@@ -133,11 +133,16 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     ) -> Result<StandardAttestationRewards, BeaconChainError> {
         let spec = &self.spec;
 
+        // Build required caches.
+        initialize_epoch_cache(&mut state, spec)?;
+        initialize_progressive_balances_cache(&mut state, spec)?;
+        state.build_exit_cache(spec)?;
+        state.build_committee_cache(RelativeEpoch::Previous, spec)?;
+        state.build_committee_cache(RelativeEpoch::Current, spec)?;
+
         // Calculate ideal_rewards
-        let participation_cache = ParticipationCache::new(&state, spec)?;
-        process_justification_and_finalization(&state, &participation_cache)?
-            .apply_changes_to_state(&mut state);
-        process_inactivity_updates(&mut state, &participation_cache, spec)?;
+        process_justification_and_finalization(&state)?.apply_changes_to_state(&mut state);
+        process_inactivity_updates_slow(&mut state, spec)?;
 
         let previous_epoch = state.previous_epoch();
 
@@ -147,18 +152,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             let weight = get_flag_weight(flag_index)
                 .map_err(|_| BeaconChainError::AttestationRewardsError)?;
 
-            let unslashed_participating_indices = participation_cache
-                .get_unslashed_participating_indices(flag_index, previous_epoch)?;
-
-            let unslashed_participating_balance =
-                unslashed_participating_indices
-                    .total_balance()
-                    .map_err(|_| BeaconChainError::AttestationRewardsError)?;
+            let unslashed_participating_balance = state
+                .progressive_balances_cache()
+                .previous_epoch_flag_attesting_balance(flag_index)?;
 
             let unslashed_participating_increments =
                 unslashed_participating_balance.safe_div(spec.effective_balance_increment)?;
 
-            let total_active_balance = participation_cache.current_epoch_total_active_balance();
+            let total_active_balance = state.get_total_active_balance()?;
 
             let active_increments =
                 total_active_balance.safe_div(spec.effective_balance_increment)?;
@@ -194,30 +195,49 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let mut total_rewards: Vec<TotalAttestationRewards> = Vec::new();
 
         let validators = if validators.is_empty() {
-            participation_cache.eligible_validator_indices().to_vec()
+            Self::all_eligible_validator_indices(&state, previous_epoch)?
         } else {
             Self::validators_ids_to_indices(&mut state, validators)?
         };
 
-        for validator_index in &validators {
-            let eligible = state.is_eligible_validator(previous_epoch, *validator_index)?;
+        for &validator_index in &validators {
+            // Return 0s for unknown/inactive validator indices.
+            let Ok(validator) = state.get_validator(validator_index) else {
+                debug!(
+                    self.log,
+                    "No rewards for inactive/unknown validator";
+                    "index" => validator_index,
+                    "epoch" => previous_epoch
+                );
+                total_rewards.push(TotalAttestationRewards {
+                    validator_index: validator_index as u64,
+                    head: 0,
+                    target: 0,
+                    source: 0,
+                    inclusion_delay: None,
+                    inactivity: 0,
+                });
+                continue;
+            };
+            let previous_epoch_participation_flags = state
+                .previous_epoch_participation()?
+                .get(validator_index)
+                .ok_or(BeaconChainError::AttestationRewardsError)?;
+            let eligible = state.is_eligible_validator(previous_epoch, validator)?;
             let mut head_reward = 0i64;
             let mut target_reward = 0i64;
             let mut source_reward = 0i64;
             let mut inactivity_penalty = 0i64;
 
             if eligible {
-                let effective_balance = state.get_effective_balance(*validator_index)?;
+                let effective_balance = validator.effective_balance;
 
                 for flag_index in 0..PARTICIPATION_FLAG_WEIGHTS.len() {
                     let (ideal_reward, penalty) = ideal_rewards_hashmap
                         .get(&(flag_index, effective_balance))
                         .ok_or(BeaconChainError::AttestationRewardsError)?;
-                    let voted_correctly = participation_cache
-                        .get_unslashed_participating_indices(flag_index, previous_epoch)
-                        .map_err(|_| BeaconChainError::AttestationRewardsError)?
-                        .contains(*validator_index)
-                        .map_err(|_| BeaconChainError::AttestationRewardsError)?;
+                    let voted_correctly = !validator.slashed
+                        && previous_epoch_participation_flags.has_flag(flag_index)?;
                     if voted_correctly {
                         if flag_index == TIMELY_HEAD_FLAG_INDEX {
                             head_reward += *ideal_reward as i64;
@@ -232,10 +252,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                         target_reward = *penalty;
 
                         let penalty_numerator = effective_balance
-                            .safe_mul(state.get_inactivity_score(*validator_index)?)?;
-                        let penalty_denominator = spec
-                            .inactivity_score_bias
-                            .safe_mul(spec.inactivity_penalty_quotient_for_state(&state))?;
+                            .safe_mul(state.get_inactivity_score(validator_index)?)?;
+                        let penalty_denominator = spec.inactivity_score_bias.safe_mul(
+                            spec.inactivity_penalty_quotient_for_fork(state.fork_name_unchecked()),
+                        )?;
                         inactivity_penalty =
                             -(penalty_numerator.safe_div(penalty_denominator)? as i64);
                     } else if flag_index == TIMELY_SOURCE_FLAG_INDEX {
@@ -244,7 +264,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 }
             }
             total_rewards.push(TotalAttestationRewards {
-                validator_index: *validator_index as u64,
+                validator_index: validator_index as u64,
                 head: head_reward,
                 target: target_reward,
                 source: source_reward,
@@ -301,6 +321,24 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         Ok(max_steps)
     }
 
+    fn all_eligible_validator_indices(
+        state: &BeaconState<T::EthSpec>,
+        previous_epoch: Epoch,
+    ) -> Result<Vec<usize>, BeaconChainError> {
+        state
+            .validators()
+            .iter()
+            .enumerate()
+            .filter_map(|(i, validator)| {
+                state
+                    .is_eligible_validator(previous_epoch, validator)
+                    .map(|eligible| eligible.then_some(i))
+                    .map_err(BeaconChainError::BeaconStateError)
+                    .transpose()
+            })
+            .collect()
+    }
+
     fn validators_ids_to_indices(
         state: &mut BeaconState<T::EthSpec>,
         validators: Vec<ValidatorId>,
@@ -339,15 +377,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         };
 
         let mut ideal_attestation_rewards_list = Vec::new();
-
+        let sqrt_total_active_balance = SqrtTotalActiveBalance::new(total_balances.current_epoch());
         for effective_balance_step in 1..=self.max_effective_balance_increment_steps()? {
             let effective_balance =
                 effective_balance_step.safe_mul(spec.effective_balance_increment)?;
-            let base_reward = get_base_reward_from_effective_balance::<T::EthSpec>(
-                effective_balance,
-                total_balances.current_epoch(),
-                spec,
-            )?;
+            let base_reward =
+                base::get_base_reward(effective_balance, sqrt_total_active_balance, spec)?;
 
             // compute ideal head rewards
             let head = get_attestation_component_delta(

beacon_node/beacon_chain/src/attestation_verification.rs

@@ -1065,7 +1065,7 @@ pub fn verify_propagation_slot_range<S: SlotClock, E: EthSpec>(
     let earliest_permissible_slot = match current_fork {
         ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => one_epoch_prior,
         // EIP-7045
-        ForkName::Deneb => one_epoch_prior
+        ForkName::Deneb | ForkName::Electra => one_epoch_prior
             .epoch(E::slots_per_epoch())
             .start_slot(E::slots_per_epoch()),
     };
@@ -1121,13 +1121,13 @@ pub fn verify_attestation_signature<T: BeaconChainTypes>(
 
 /// Verifies that the `attestation.data.target.root` is indeed the target root of the block at
 /// `attestation.data.beacon_block_root`.
-pub fn verify_attestation_target_root<T: EthSpec>(
+pub fn verify_attestation_target_root<E: EthSpec>(
     head_block: &ProtoBlock,
-    attestation: &Attestation<T>,
+    attestation: &Attestation<E>,
 ) -> Result<(), Error> {
     // Check the attestation target root.
-    let head_block_epoch = head_block.slot.epoch(T::slots_per_epoch());
-    let attestation_epoch = attestation.data.slot.epoch(T::slots_per_epoch());
+    let head_block_epoch = head_block.slot.epoch(E::slots_per_epoch());
+    let attestation_epoch = attestation.data.slot.epoch(E::slots_per_epoch());
     if head_block_epoch > attestation_epoch {
         // The epoch references an invalid head block from a future epoch.
         //