feat: implement bal cache as a part of BlockChainProvider #22952

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed

Soubhik-10 wants to merge 43 commits into paradigmxyz:bal-devnet-3 from Soubhik-10:bal-more-rpc

.github/scripts/hive/expected_failures.yaml

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -20,11 +20,11 @@ engine-withdrawals: []
  
    engine-api: []

    engine-cancun:

      - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)

    engine-cancun: []

      # - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)

      # the test fails with older versions of the code for which it passed before, probably related to changes

      # in hive or its dependencies

      - Blob Transaction Ordering, Multiple Clients (Cancun) (reth)

      # - Blob Transaction Ordering, Multiple Clients (Cancun) (reth)

    sync: []

Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Cargo.toml

-Original file line number
+Diff line change
@@ Expand Up / @@ -49,6 +49,7 @@ members = [ @@
         "crates/exex/test-utils/",
         "crates/exex/types/",
         "crates/metrics/",
+        "crates/net/bal-store/",
         "crates/net/banlist/",
         "crates/net/discv4/",
         "crates/net/discv5/",
@@ Expand Down Expand Up / @@ -375,6 +376,7 @@ reth-ipc = { path = "crates/rpc/ipc" } @@
     reth-libmdbx = { path = "crates/storage/libmdbx-rs" }
     reth-mdbx-sys = { path = "crates/storage/libmdbx-rs/mdbx-sys" }
     reth-metrics = { path = "crates/metrics" }
+    reth-bal-store = { path = "crates/net/bal-store" }
     reth-net-banlist = { path = "crates/net/banlist" }
     reth-net-nat = { path = "crates/net/nat" }
     reth-network = { path = "crates/net/network" }
@@ Expand Down @@

crates/engine/tree/Cargo.toml

-Original file line number
+Diff line change
@@ Expand Up / @@ -12,6 +12,7 @@ workspace = true @@
     [dependencies]
     # reth
+    reth-bal-store.workspace = true
     reth-chain-state = { workspace = true, features = ["rayon"] }
     reth-chainspec = { workspace = true, optional = true }
     reth-consensus.workspace = true
@@ Expand Down @@

crates/engine/tree/src/tree/payload_processor/mod.rs

-Original file line number
+Diff line change
@@ Expand Up / @@ -1178,7 +1178,7 @@ mod tests { @@
         use reth_evm_ethereum::EthEvmConfig;
         use reth_primitives_traits::{Account, Recovered, StorageEntry};
         use reth_provider::{
-            providers::{BlockchainProvider, OverlayStateProviderFactory},
+            providers::{BalProvider, BlockchainProvider, OverlayStateProviderFactory},
             test_utils::create_test_provider_factory_with_chain_spec,
             ChainSpecProvider, HashingWriter,
         };
@@ Expand Down Expand Up / @@ -1444,7 +1444,7 @@ mod tests { @@
                 PrecompileCacheMap::default(),
             );
-            let provider_factory = BlockchainProvider::new(factory).unwrap();
+            let provider_factory = BlockchainProvider::new(factory, BalProvider::default()).unwrap();
             let mut handle = payload_processor.spawn(
                 ExecutionEnv::test_default(),
@@ Expand Down @@

crates/engine/tree/src/tree/payload_validator.rs

-Original file line number
+Diff line change
@@ Expand Up @@
     use alloy_evm::Evm;
     use alloy_primitives::{map::B256Set, B256};
     use alloy_rlp::Decodable;
+    use reth_bal_store::BalStore;
     #[cfg(feature = "trie-debug")]
     use reth_trie_sparse::debug_recorder::TrieDebugRecorder;
@@ Expand Down Expand Up / @@ -178,6 +179,7 @@ where @@
             + StateProviderFactory
             + StateReader
             + HashedPostStateProvider
+            + BalStore
             + Clone
             + 'static,
         Evm: ConfigureEvm<Primitives = N> + 'static,
@@ Expand Down Expand Up / @@ -1387,13 +1389,18 @@ where @@
                 block,
                 output,
                 receipt_root_bloom,
-                built_bal,
+                built_bal.clone(),
                 true,
             ) {
                 // call post-block hook
                 self.on_invalid_block(parent_block, block, output, None, ctx.state_mut());
                 return Err(err.into())
             }
+            if built_bal.is_some() {
+                let bal_bytes: alloy_primitives::Bytes =
+                    alloy_rlp::encode(built_bal.unwrap_or_default()).into();
+                let _ = self.provider.insert(block.hash(), block.num_hash().number, bal_bytes);
+            }
             drop(_enter);
             // Wait for the background keccak256 hashing task to complete. This blocks until
@@ Expand Down Expand Up / @@ -1968,6 +1975,7 @@ where @@
             + ChangeSetReader
             + BlockNumReader
             + HashedPostStateProvider
+            + BalStore
             + Clone
             + 'static,
         N: NodePrimitives,
@@ Expand Down @@

crates/exex/exex/src/backfill/job.rs

-Original file line number
+Diff line change
@@ Expand Up / @@ -253,7 +253,8 @@ mod tests { @@
         use reth_evm_ethereum::EthEvmConfig;
         use reth_primitives_traits::crypto::secp256k1::public_key_to_address;
         use reth_provider::{
-            providers::BlockchainProvider, test_utils::create_test_provider_factory_with_chain_spec,
+            providers::{BalProvider, BlockchainProvider},
+            test_utils::create_test_provider_factory_with_chain_spec,
         };
         use reth_testing_utils::generators;
@@ Expand All / @@ -270,7 +271,8 @@ mod tests { @@
             let executor = EthEvmConfig::ethereum(chain_spec.clone());
             let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
             init_genesis(&provider_factory)?;
-            let blockchain_db = BlockchainProvider::new(provider_factory.clone())?;
+            let blockchain_db =
+                BlockchainProvider::new(provider_factory.clone(), BalProvider::default())?;
             let blocks_and_execution_outputs =
                 blocks_and_execution_outputs(provider_factory, chain_spec, key_pair)?;
@@ Expand Down Expand Up / @@ -306,7 +308,8 @@ mod tests { @@
             let executor = EthEvmConfig::ethereum(chain_spec.clone());
             let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
             init_genesis(&provider_factory)?;
-            let blockchain_db = BlockchainProvider::new(provider_factory.clone())?;
+            let blockchain_db =
+                BlockchainProvider::new(provider_factory.clone(), BalProvider::default())?;
             let blocks_and_execution_outcomes =
                 blocks_and_execution_outputs(provider_factory, chain_spec, key_pair)?;
@@ Expand Down Expand Up / @@ -356,7 +359,8 @@ mod tests { @@
             let executor = EthEvmConfig::ethereum(chain_spec.clone());
             let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
             init_genesis(&provider_factory)?;
-            let blockchain_db = BlockchainProvider::new(provider_factory.clone())?;
+            let blockchain_db =
+                BlockchainProvider::new(provider_factory.clone(), BalProvider::default())?;
             // Execute blocks via LatestStateProvider (pipeline-style) and commit to DB.
             // This mirrors what the pipeline's ExecutionStage does.
@@ Expand Down Expand Up / @@ -420,7 +424,8 @@ mod tests { @@
             let executor = EthEvmConfig::ethereum(chain_spec.clone());
             let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
             init_genesis(&provider_factory)?;
-            let blockchain_db = BlockchainProvider::new(provider_factory.clone())?;
+            let blockchain_db =
+                BlockchainProvider::new(provider_factory.clone(), BalProvider::default())?;
             let pipeline_results =
                 blocks_and_execution_outputs(provider_factory, chain_spec, key_pair)?;
@@ Expand Down Expand Up / @@ -463,7 +468,8 @@ mod tests { @@
             let executor = EthEvmConfig::ethereum(chain_spec.clone());
             let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
             init_genesis(&provider_factory)?;
-            let blockchain_db = BlockchainProvider::new(provider_factory.clone())?;
+            let blockchain_db =
+                BlockchainProvider::new(provider_factory.clone(), BalProvider::default())?;
             let blocks_and_execution_outputs =
                 blocks_and_execution_outputs(provider_factory, chain_spec, key_pair)?;
@@ Expand Down @@

crates/exex/exex/src/backfill/stream.rs

-Original file line number
+Diff line change
@@ Expand Up / @@ -259,7 +259,7 @@ mod tests { @@
             crypto::secp256k1::public_key_to_address, Block as _, NodePrimitives,
         };
         use reth_provider::{
-            providers::{BlockchainProvider, ProviderNodeTypes},
+            providers::{BalProvider, BlockchainProvider, ProviderNodeTypes},
             test_utils::create_test_provider_factory_with_chain_spec,
             ProviderFactory,
         };
@@ Expand All / @@ -281,7 +281,8 @@ mod tests { @@
             let executor = EthEvmConfig::ethereum(chain_spec.clone());
             let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
             init_genesis(&provider_factory)?;
-            let blockchain_db = BlockchainProvider::new(provider_factory.clone())?;
+            let blockchain_db =
+                BlockchainProvider::new(provider_factory.clone(), BalProvider::default())?;
             // Create first 2 blocks
             let blocks_and_execution_outcomes =
@@ Expand Down Expand Up / @@ -318,7 +319,8 @@ mod tests { @@
             let executor = EthEvmConfig::ethereum(chain_spec.clone());
             let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
             init_genesis(&provider_factory)?;
-            let blockchain_db = BlockchainProvider::new(provider_factory.clone())?;
+            let blockchain_db =
+                BlockchainProvider::new(provider_factory.clone(), BalProvider::default())?;
             // Create first 2 blocks
             let (blocks, execution_outcome) =
@@ Expand Down Expand Up / @@ -421,7 +423,8 @@ mod tests { @@
             let executor = EthEvmConfig::ethereum(chain_spec.clone());
             let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
             init_genesis(&provider_factory)?;
-            let blockchain_db = BlockchainProvider::new(provider_factory.clone())?;
+            let blockchain_db =
+                BlockchainProvider::new(provider_factory.clone(), BalProvider::default())?;
             // Create and commit 4 blocks
             let blocks = create_blocks(&chain_spec, key_pair, 4)?;
@@ Expand Down @@

crates/exex/exex/src/manager.rs

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -687,8 +687,9 @@ mod tests {
  
        use reth_evm_ethereum::EthEvmConfig;

        use reth_primitives_traits::RecoveredBlock;

        use reth_provider::{

            providers::BlockchainProvider, test_utils::create_test_provider_factory, BlockReader,

            BlockWriter, Chain, DBProvider, DatabaseProviderFactory, TransactionVariant,

            providers::{BalProvider, BlockchainProvider},

            test_utils::create_test_provider_factory,

            BlockReader, BlockWriter, Chain, DBProvider, DatabaseProviderFactory, TransactionVariant,

        };

        use reth_testing_utils::generators::{self, random_block, BlockParams};

    @@ -1119,7 +1120,7 @@ mod tests {
  
        async fn exex_handle_new() {

            let provider_factory = create_test_provider_factory();

            init_genesis(&provider_factory).unwrap();

            let provider = BlockchainProvider::new(provider_factory).unwrap();

            let provider = BlockchainProvider::new(provider_factory, BalProvider::default()).unwrap();

            let temp_dir = tempfile::tempdir().unwrap();

            let wal = Wal::new(temp_dir.path()).unwrap();

    @@ -1174,7 +1175,7 @@ mod tests {
  
        async fn test_notification_if_finished_height_gt_chain_tip() {

            let provider_factory = create_test_provider_factory();

            init_genesis(&provider_factory).unwrap();

            let provider = BlockchainProvider::new(provider_factory).unwrap();

            let provider = BlockchainProvider::new(provider_factory, BalProvider::default()).unwrap();

            let temp_dir = tempfile::tempdir().unwrap();

            let wal = Wal::new(temp_dir.path()).unwrap();

    @@ -1224,7 +1225,7 @@ mod tests {
  
        async fn test_sends_chain_reorged_notification() {

            let provider_factory = create_test_provider_factory();

            init_genesis(&provider_factory).unwrap();

            let provider = BlockchainProvider::new(provider_factory).unwrap();

            let provider = BlockchainProvider::new(provider_factory, BalProvider::default()).unwrap();

            let temp_dir = tempfile::tempdir().unwrap();

            let wal = Wal::new(temp_dir.path()).unwrap();

    @@ -1267,7 +1268,7 @@ mod tests {
  
        async fn test_sends_chain_reverted_notification() {

            let provider_factory = create_test_provider_factory();

            init_genesis(&provider_factory).unwrap();

            let provider = BlockchainProvider::new(provider_factory).unwrap();

            let provider = BlockchainProvider::new(provider_factory, BalProvider::default()).unwrap();

            let temp_dir = tempfile::tempdir().unwrap();

            let wal = Wal::new(temp_dir.path()).unwrap();

    @@ -1327,7 +1328,7 @@ mod tests {
  
            provider_rw.insert_block(&block).unwrap();

            provider_rw.commit().unwrap();

            let provider = BlockchainProvider::new(provider_factory).unwrap();

            let provider = BlockchainProvider::new(provider_factory, BalProvider::default()).unwrap();

            let temp_dir = tempfile::tempdir().unwrap();

            let wal = Wal::new(temp_dir.path()).unwrap();

    @@ -1428,7 +1429,8 @@ mod tests {
  
            let wal = Wal::new(temp_dir.path()).unwrap();

            let provider_factory = create_test_provider_factory();

            init_genesis(&provider_factory).unwrap();

            let provider = BlockchainProvider::new(provider_factory.clone()).unwrap();

            let provider =

                BlockchainProvider::new(provider_factory.clone(), BalProvider::default()).unwrap();

            // 1. Setup Manager with Capacity = 1

            let (exex_handle, _, mut notifications) = ExExHandle::new(

crates/exex/exex/src/notifications.rs

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -529,8 +529,9 @@ mod tests {
  
        use reth_evm_ethereum::EthEvmConfig;

        use reth_primitives_traits::Block as _;

        use reth_provider::{

            providers::BlockchainProvider, test_utils::create_test_provider_factory, BlockWriter,

            Chain, DBProvider, DatabaseProviderFactory,

            providers::{BalProvider, BlockchainProvider},

            test_utils::create_test_provider_factory,

            BlockWriter, Chain, DBProvider, DatabaseProviderFactory,

        };

        use reth_testing_utils::generators::{self, random_block, BlockParams};

        use std::collections::BTreeMap;

    @@ -549,7 +550,7 @@ mod tests {
  
                .block(genesis_hash.into())?

                .ok_or_else(|| eyre::eyre!("genesis block not found"))?;

            let provider = BlockchainProvider::new(provider_factory.clone())?;

            let provider = BlockchainProvider::new(provider_factory.clone(), BalProvider::default())?;

            let node_head_block = random_block(

                &mut rng,

    @@ -623,7 +624,7 @@ mod tests {
  
                .block(genesis_hash.into())?

                .ok_or_else(|| eyre::eyre!("genesis block not found"))?;

            let provider = BlockchainProvider::new(provider_factory)?;

            let provider = BlockchainProvider::new(provider_factory, BalProvider::default())?;

            let node_head = BlockNumHash { number: genesis_block.number, hash: genesis_hash };

            let exex_head = ExExHead { block: node_head };

    @@ -677,7 +678,7 @@ mod tests {
  
                .block(genesis_hash.into())?

                .ok_or_else(|| eyre::eyre!("genesis block not found"))?;

            let provider = BlockchainProvider::new(provider_factory)?;

            let provider = BlockchainProvider::new(provider_factory, BalProvider::default())?;

            let node_head_block = random_block(

                &mut rng,

    @@ -768,7 +769,7 @@ mod tests {
  
                .block(genesis_hash.into())?

                .ok_or_else(|| eyre::eyre!("genesis block not found"))?;

            let provider = BlockchainProvider::new(provider_factory)?;

            let provider = BlockchainProvider::new(provider_factory, BalProvider::default())?;

            let exex_head_block = random_block(

                &mut rng,

    @@ -853,7 +854,7 @@ mod tests {
  
                .block(genesis_hash.into())?

                .ok_or_else(|| eyre::eyre!("genesis block not found"))?;

            let provider = BlockchainProvider::new(provider_factory.clone())?;

            let provider = BlockchainProvider::new(provider_factory.clone(), BalProvider::default())?;

            // Insert block 1 into the DB so there's something to backfill

            let node_head_block = random_block(

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat: implement bal cache as a part of BlockChainProvider #22952

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!