feat(storage): read headers and transactions only from static files

crates/engine/tree/src/tree/mod.rs

@@ -2659,7 +2659,7 @@ where
         let mut canonical = self.canonical_in_memory_state.header_by_hash(hash);
 
         if canonical.is_none() {
-            canonical = self.provider.header(&hash)?.map(|header| SealedHeader::new(header, hash));
+            canonical = self.provider.header(hash)?.map(|header| SealedHeader::new(header, hash));
         }
 
         Ok(canonical)
@@ -2883,7 +2883,7 @@ where
         }
 
         // Check if the block is persisted
-        if let Some(header) = self.provider.header(&hash)? {
+        if let Some(header) = self.provider.header(hash)? {
             debug!(target: "engine::tree", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder");
             // For persisted blocks, we create a builder that will fetch state directly from the
             // database

crates/engine/tree/src/tree/payload_validator.rs

@@ -967,7 +967,7 @@ where
         }
 
         // Check if the block is persisted
-        if let Some(header) = self.provider.header(&hash)? {
+        if let Some(header) = self.provider.header(hash)? {
             debug!(target: "engine::tree", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder");
             // For persisted blocks, we create a builder that will fetch state directly from the
             // database

crates/exex/exex/src/manager.rs

@@ -370,7 +370,7 @@ where
             .map(|(exex_id, num_hash)| {
                 num_hash.map_or(Ok((exex_id, num_hash, false)), |num_hash| {
                     self.provider
-                        .is_known(&num_hash.hash)
+                        .is_known(num_hash.hash)
                         // Save the ExEx ID, finished height, and whether the hash is canonical
                         .map(|is_canonical| (exex_id, Some(num_hash), is_canonical))
                 })

crates/exex/exex/src/notifications.rs

@@ -308,7 +308,7 @@ where
     /// we're not on the canonical chain and we need to revert the notification with the ExEx
     /// head block.
     fn check_canonical(&mut self) -> eyre::Result<Option<ExExNotification<E::Primitives>>> {
-        if self.provider.is_known(&self.initial_exex_head.block.hash)? &&
+        if self.provider.is_known(self.initial_exex_head.block.hash)? &&
             self.initial_exex_head.block.number <= self.initial_local_head.number
         {
             // we have the targeted block and that block is below the current head

crates/net/downloaders/Cargo.toml

@@ -22,9 +22,8 @@ reth-storage-api.workspace = true
 reth-tasks.workspace = true
 
 # optional deps for the test-utils feature
-reth-db = { workspace = true, optional = true }
-reth-db-api = { workspace = true, optional = true }
 reth-ethereum-primitives = { workspace = true, optional = true }
+reth-provider = { workspace = true, optional = true }
 reth-testing-utils = { workspace = true, optional = true }
 
 # ethereum
@@ -58,8 +57,6 @@ itertools.workspace = true
 async-compression = { workspace = true, features = ["gzip", "tokio"] }
 reth-ethereum-primitives.workspace = true
 reth-chainspec.workspace = true
-reth-db = { workspace = true, features = ["test-utils"] }
-reth-db-api.workspace = true
 reth-consensus = { workspace = true, features = ["test-utils"] }
 reth-network-p2p = { workspace = true, features = ["test-utils"] }
 reth-provider = { workspace = true, features = ["test-utils"] }
@@ -76,13 +73,10 @@ default = []
 file-client = ["dep:async-compression", "dep:alloy-rlp"]
 test-utils = [
     "tempfile",
-    "reth-db-api",
-    "reth-db/test-utils",
     "reth-consensus/test-utils",
     "reth-network-p2p/test-utils",
     "reth-testing-utils",
     "reth-chainspec/test-utils",
-    "reth-db-api?/test-utils",
     "reth-provider/test-utils",
     "reth-primitives-traits/test-utils",
     "dep:reth-ethereum-primitives",

crates/net/downloaders/src/bodies/bodies.rs

@@ -618,12 +618,8 @@ mod tests {
     };
     use alloy_primitives::B256;
     use assert_matches::assert_matches;
-    use reth_chainspec::MAINNET;
     use reth_consensus::test_utils::TestConsensus;
-    use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir};
-    use reth_provider::{
-        providers::StaticFileProvider, test_utils::MockNodeTypesWithDB, ProviderFactory,
-    };
+    use reth_provider::test_utils::create_test_provider_factory;
     use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams};
     use std::collections::HashMap;
 
@@ -632,25 +628,20 @@ mod tests {
     #[tokio::test]
     async fn streams_bodies_in_order() {
         // Generate some random blocks
-        let db = create_test_rw_db();
+        let factory = create_test_provider_factory();
         let (headers, mut bodies) = generate_bodies(0..=19);
 
-        insert_headers(db.db(), &headers);
+        insert_headers(&factory, &headers);
 
         let client = Arc::new(
             TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true),
         );
-        let (_static_dir, static_dir_path) = create_test_static_files_dir();
 
         let mut downloader = BodiesDownloaderBuilder::default()
             .build::<reth_ethereum_primitives::Block, _, _>(
                 client.clone(),
                 Arc::new(TestConsensus::default()),
-                ProviderFactory::<MockNodeTypesWithDB>::new(
-                    db,
-                    MAINNET.clone(),
-                    StaticFileProvider::read_write(static_dir_path).unwrap(),
-                ),
+                factory,
             );
         downloader.set_download_range(0..=19).expect("failed to set download range");
 
@@ -666,7 +657,7 @@ mod tests {
     #[tokio::test]
     async fn requests_correct_number_of_times() {
         // Generate some random blocks
-        let db = create_test_rw_db();
+        let factory = create_test_provider_factory();
         let mut rng = generators::rng();
         let blocks = random_block_range(
             &mut rng,
@@ -680,22 +671,17 @@ mod tests {
             .map(|block| (block.hash(), block.into_body()))
             .collect::<HashMap<_, _>>();
 
-        insert_headers(db.db(), &headers);
+        insert_headers(&factory, &headers);
 
         let request_limit = 10;
         let client = Arc::new(TestBodiesClient::default().with_bodies(bodies.clone()));
-        let (_static_dir, static_dir_path) = create_test_static_files_dir();
 
         let mut downloader = BodiesDownloaderBuilder::default()
             .with_request_limit(request_limit)
             .build::<reth_ethereum_primitives::Block, _, _>(
             client.clone(),
             Arc::new(TestConsensus::default()),
-            ProviderFactory::<MockNodeTypesWithDB>::new(
-                db,
-                MAINNET.clone(),
-                StaticFileProvider::read_write(static_dir_path).unwrap(),
-            ),
+            factory,
         );
         downloader.set_download_range(0..=199).expect("failed to set download range");
 
@@ -708,28 +694,23 @@ mod tests {
     #[tokio::test]
     async fn streams_bodies_in_order_after_range_reset() {
         // Generate some random blocks
-        let db = create_test_rw_db();
+        let factory = create_test_provider_factory();
         let (headers, mut bodies) = generate_bodies(0..=99);
 
-        insert_headers(db.db(), &headers);
+        insert_headers(&factory, &headers);
 
         let stream_batch_size = 20;
         let request_limit = 10;
         let client = Arc::new(
             TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true),
         );
-        let (_static_dir, static_dir_path) = create_test_static_files_dir();
         let mut downloader = BodiesDownloaderBuilder::default()
             .with_stream_batch_size(stream_batch_size)
             .with_request_limit(request_limit)
             .build::<reth_ethereum_primitives::Block, _, _>(
                 client.clone(),
                 Arc::new(TestConsensus::default()),
-                ProviderFactory::<MockNodeTypesWithDB>::new(
-                    db,
-                    MAINNET.clone(),
-                    StaticFileProvider::read_write(static_dir_path).unwrap(),
-                ),
+                factory,
             );
 
         let mut range_start = 0;
@@ -750,24 +731,19 @@ mod tests {
     #[tokio::test]
     async fn can_download_new_range_after_termination() {
         // Generate some random blocks
-        let db = create_test_rw_db();
+        let factory = create_test_provider_factory();
         let (headers, mut bodies) = generate_bodies(0..=199);
 
-        insert_headers(db.db(), &headers);
+        insert_headers(&factory, &headers);
 
         let client = Arc::new(TestBodiesClient::default().with_bodies(bodies.clone()));
-        let (_static_dir, static_dir_path) = create_test_static_files_dir();
 
         let mut downloader = BodiesDownloaderBuilder::default()
             .with_stream_batch_size(100)
             .build::<reth_ethereum_primitives::Block, _, _>(
             client.clone(),
             Arc::new(TestConsensus::default()),
-            ProviderFactory::<MockNodeTypesWithDB>::new(
-                db,
-                MAINNET.clone(),
-                StaticFileProvider::read_write(static_dir_path).unwrap(),
-            ),
+            factory,
         );
 
         // Set and download the first range
@@ -792,14 +768,13 @@ mod tests {
     #[tokio::test]
     async fn can_download_after_exceeding_limit() {
         // Generate some random blocks
-        let db = create_test_rw_db();
+        let factory = create_test_provider_factory();
         let (headers, mut bodies) = generate_bodies(0..=199);
 
-        insert_headers(db.db(), &headers);
+        insert_headers(&factory, &headers);
 
         let client = Arc::new(TestBodiesClient::default().with_bodies(bodies.clone()));
 
-        let (_static_dir, static_dir_path) = create_test_static_files_dir();
         // Set the max buffered block size to 1 byte, to make sure that every response exceeds the
         // limit
         let mut downloader = BodiesDownloaderBuilder::default()
@@ -809,11 +784,7 @@ mod tests {
             .build::<reth_ethereum_primitives::Block, _, _>(
                 client.clone(),
                 Arc::new(TestConsensus::default()),
-                ProviderFactory::<MockNodeTypesWithDB>::new(
-                    db,
-                    MAINNET.clone(),
-                    StaticFileProvider::read_write(static_dir_path).unwrap(),
-                ),
+                factory,
             );
 
         // Set and download the entire range
@@ -829,28 +800,23 @@ mod tests {
     #[tokio::test]
     async fn can_tolerate_empty_responses() {
         // Generate some random blocks
-        let db = create_test_rw_db();
+        let factory = create_test_provider_factory();
         let (headers, mut bodies) = generate_bodies(0..=99);
 
-        insert_headers(db.db(), &headers);
+        insert_headers(&factory, &headers);
 
         // respond with empty bodies for every other request.
         let client = Arc::new(
             TestBodiesClient::default().with_bodies(bodies.clone()).with_empty_responses(2),
         );
-        let (_static_dir, static_dir_path) = create_test_static_files_dir();
 
         let mut downloader = BodiesDownloaderBuilder::default()
             .with_request_limit(3)
             .with_stream_batch_size(100)
             .build::<reth_ethereum_primitives::Block, _, _>(
                 client.clone(),
                 Arc::new(TestConsensus::default()),
-                ProviderFactory::<MockNodeTypesWithDB>::new(
-                    db,
-                    MAINNET.clone(),
-                    StaticFileProvider::read_write(static_dir_path).unwrap(),
-                ),
+                factory,
             );
 
         // Download the requested range

crates/net/downloaders/src/bodies/task.rs

@@ -190,7 +190,7 @@ mod tests {
         let factory = create_test_provider_factory();
         let (headers, mut bodies) = generate_bodies(0..=19);
 
-        insert_headers(factory.db_ref().db(), &headers);
+        insert_headers(&factory, &headers);
 
         let client = Arc::new(
             TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true),

crates/net/downloaders/src/bodies/test_utils.rs

@@ -3,12 +3,14 @@
 #![allow(dead_code)]
 
 use alloy_consensus::BlockHeader;
-use alloy_primitives::B256;
-use reth_db::DatabaseEnv;
-use reth_db_api::{database::Database, tables, transaction::DbTxMut};
+use alloy_primitives::{B256, U256};
 use reth_ethereum_primitives::BlockBody;
 use reth_network_p2p::bodies::response::BlockResponse;
 use reth_primitives_traits::{Block, SealedBlock, SealedHeader};
+use reth_provider::{
+    test_utils::MockNodeTypesWithDB, ProviderFactory, StaticFileProviderFactory, StaticFileSegment,
+    StaticFileWriter,
+};
 use std::collections::HashMap;
 
 pub(crate) fn zip_blocks<'a, B: Block>(
@@ -42,12 +44,21 @@ pub(crate) fn create_raw_bodies(
 }
 
 #[inline]
-pub(crate) fn insert_headers(db: &DatabaseEnv, headers: &[SealedHeader]) {
-    db.update(|tx| {
-        for header in headers {
-            tx.put::<tables::CanonicalHeaders>(header.number, header.hash()).unwrap();
-            tx.put::<tables::Headers>(header.number, header.clone_header()).unwrap();
-        }
-    })
-    .expect("failed to commit")
+pub(crate) fn insert_headers(
+    factory: &ProviderFactory<MockNodeTypesWithDB>,
+    headers: &[SealedHeader],
+) {
+    let provider_rw = factory.provider_rw().expect("failed to create provider");
+    let static_file_provider = provider_rw.static_file_provider();
+    let mut writer = static_file_provider
+        .latest_writer(StaticFileSegment::Headers)
+        .expect("failed to create writer");
+
+    for header in headers {
+        writer
+            .append_header(header.header(), U256::ZERO, &header.hash())
+            .expect("failed to append header");
+    }
+    drop(writer);
+    provider_rw.commit().expect("failed to commit");
 }

crates/net/downloaders/src/file_client.rs

@@ -675,7 +675,7 @@ mod tests {
         let factory = create_test_provider_factory();
         let (headers, mut bodies) = generate_bodies(0..=19);
 
-        insert_headers(factory.db_ref().db(), &headers);
+        insert_headers(&factory, &headers);
 
         // create an empty file
         let file = tempfile::tempfile().unwrap();
@@ -770,7 +770,7 @@ mod tests {
             Arc::new(FileClient::from_file(file, NoopConsensus::arc()).await.unwrap());
 
         // insert headers in db for the bodies downloader
-        insert_headers(factory.db_ref().db(), &headers);
+        insert_headers(&factory, &headers);
 
         let mut downloader = BodiesDownloaderBuilder::default().build::<Block, _, _>(
             client.clone(),

crates/prune/prune/src/segments/user/transaction_lookup.rs

@@ -157,7 +157,7 @@ mod tests {
             1..=10,
             BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() },
         );
-        db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
+        db.insert_blocks(blocks.iter(), StorageKind::Static).expect("insert blocks");
 
         let mut tx_hash_numbers = Vec::new();
         for block in &blocks {
@@ -170,11 +170,11 @@ mod tests {
         db.insert_tx_hash_numbers(tx_hash_numbers).expect("insert tx hash numbers");
 
         assert_eq!(
-            db.table::<tables::Transactions>().unwrap().len(),
+            db.count_entries::<tables::Transactions>().unwrap(),
             blocks.iter().map(|block| block.transaction_count()).sum::<usize>()
         );
         assert_eq!(
-            db.table::<tables::Transactions>().unwrap().len(),
+            db.count_entries::<tables::Transactions>().unwrap(),
             db.table::<tables::TransactionHashNumbers>().unwrap().len()
         );
 

crates/rpc/rpc-eth-types/src/cache/mod.rs

@@ -539,7 +539,7 @@ where
                                 this.action_task_spawner.spawn_blocking(Box::pin(async move {
                                     // Acquire permit
                                     let _permit = rate_limiter.acquire().await;
-                                    let header = provider.header(&block_hash).and_then(|header| {
+                                    let header = provider.header(block_hash).and_then(|header| {
                                         header.ok_or_else(|| {
                                             ProviderError::HeaderNotFound(block_hash.into())
                                         })