From df03d0668af65fa87d331e67613a2cbf0e10e0e5 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 9 Apr 2026 14:00:16 +0000 Subject: [PATCH 01/11] =?UTF-8?q?feat(cli):=20add=20`reth=20db=20migrate-v?= =?UTF-8?q?2`=20for=20v1=E2=86=92v2=20storage=20migration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019d725d-8f9f-740f-abae-cea809eeb511 --- crates/cli/commands/src/db/migrate_v2.rs | 446 +++++++++++++++++++++++ crates/cli/commands/src/db/mod.rs | 9 + 2 files changed, 455 insertions(+) create mode 100644 crates/cli/commands/src/db/migrate_v2.rs diff --git a/crates/cli/commands/src/db/migrate_v2.rs b/crates/cli/commands/src/db/migrate_v2.rs new file mode 100644 index 00000000000..6892169978d --- /dev/null +++ b/crates/cli/commands/src/db/migrate_v2.rs @@ -0,0 +1,446 @@ +//! `reth db migrate-v2` command for migrating v1 storage layout to v2. +//! +//! Migrates data from MDBX-only (v1) storage layout to the hybrid v2 layout: +//! - TransactionSenders → static files +//! - AccountChangeSets → static files +//! - StorageChangeSets → static files +//! - Receipts → static files (if not already there) +//! - TransactionHashNumbers → RocksDB +//! - AccountsHistory → RocksDB +//! - StoragesHistory → RocksDB +//! +//! Then updates `StorageSettings` to v2. + +use clap::Parser; +use reth_db::models::StorageBeforeTx; +use reth_db_api::{ + cursor::DbCursorRO, + database::Database, + table::Table, + tables, + transaction::{DbTx, DbTxMut}, +}; +use reth_db_common::DbTool; +use reth_provider::{ + providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory, MetadataProvider, + MetadataWriter, RocksDBProviderFactory, StaticFileProviderFactory, StaticFileWriter, + StorageSettings, +}; +use reth_stages_types::StageId; +use reth_static_file_types::StaticFileSegment; +use reth_storage_api::StageCheckpointReader; +use tracing::info; + +/// `reth db migrate-v2` command +#[derive(Debug, Parser)] +pub struct Command { + /// Prune migrated data from MDBX tables after successful migration. + #[arg(long)] + prune_mdbx: bool, +} + +impl Command { + /// Execute the migration. + pub fn execute(self, tool: &DbTool) -> eyre::Result<()> + where + N::Primitives: reth_primitives_traits::NodePrimitives< + Receipt: reth_db_api::table::Value + reth_codecs::Compact, + >, + { + // === Phase 0: Preflight === + info!(target: "reth::cli", "Starting v1 → v2 storage migration"); + + let provider = tool.provider_factory.provider()?; + let current_settings = provider.storage_settings()?; + + if current_settings.is_some_and(|s| s.is_v2()) { + info!(target: "reth::cli", "Storage is already v2, nothing to do"); + return Ok(()); + } + + let tip = + provider.get_stage_checkpoint(StageId::Execution)?.map(|c| c.block_number).unwrap_or(0); + + info!(target: "reth::cli", tip, "Chain tip block number"); + + let sf_provider = tool.provider_factory.static_file_provider(); + + // Check that target static file segments are empty + for segment in [ + StaticFileSegment::TransactionSenders, + StaticFileSegment::AccountChangeSets, + StaticFileSegment::StorageChangeSets, + ] { + if sf_provider.get_highest_static_file_block(segment).is_some() { + eyre::bail!( + "Static file segment {segment:?} already contains data. \ + Cannot migrate — target must be empty." + ); + } + } + + // Check that RocksDB tables are empty + let rocksdb = tool.provider_factory.rocksdb_provider(); + if rocksdb.first::()?.is_some() { + eyre::bail!("RocksDB TransactionHashNumbers already contains data"); + } + if rocksdb.first::()?.is_some() { + eyre::bail!("RocksDB AccountsHistory already contains data"); + } + if rocksdb.first::()?.is_some() { + eyre::bail!("RocksDB StoragesHistory already contains data"); + } + + drop(provider); + info!(target: "reth::cli", "Preflight checks passed"); + + // === Phase 1: TransactionSenders → static files === + self.migrate_transaction_senders(tool, tip)?; + + // === Phase 2: AccountChangeSets → static files === + self.migrate_account_changesets(tool, tip)?; + + // === Phase 3: StorageChangeSets → static files === + self.migrate_storage_changesets(tool, tip)?; + + // === Phase 4: Receipts → static files === + self.migrate_receipts::(tool, tip)?; + + // === Phase 5: TransactionHashNumbers → RocksDB === + self.migrate_transaction_hash_numbers(tool)?; + + // === Phase 6: AccountsHistory → RocksDB === + self.migrate_accounts_history(tool)?; + + // === Phase 7: StoragesHistory → RocksDB === + self.migrate_storages_history(tool)?; + + // === Phase 8: Verify hashed state === + self.verify_hashed_state(tool, tip)?; + + // === Phase 9: Update metadata to v2 === + info!(target: "reth::cli", "Writing StorageSettings v2 metadata"); + let provider_rw = tool.provider_factory.database_provider_rw()?; + provider_rw.write_storage_settings(StorageSettings::v2())?; + provider_rw.commit()?; + info!(target: "reth::cli", "Storage settings updated to v2"); + + // === Phase 10: Optional MDBX pruning === + if self.prune_mdbx { + self.prune_migrated_tables(tool)?; + } + + info!(target: "reth::cli", "Migration complete!"); + Ok(()) + } + + fn migrate_transaction_senders( + &self, + tool: &DbTool, + tip: u64, + ) -> eyre::Result<()> { + info!(target: "reth::cli", "Migrating TransactionSenders → static files"); + let provider = tool.provider_factory.provider()?; + let sf_provider = tool.provider_factory.static_file_provider(); + let mut writer = sf_provider.latest_writer(StaticFileSegment::TransactionSenders)?; + + let mut sender_cursor = provider.tx_ref().cursor_read::()?; + let mut block_cursor = provider.tx_ref().cursor_read::()?; + + let mut count = 0u64; + let block_walker = block_cursor.walk(Some(0))?; + for result in block_walker { + let (block_number, body_indices) = result?; + if block_number > tip { + break; + } + writer.increment_block(block_number)?; + + let tx_range = body_indices.tx_num_range(); + if tx_range.is_empty() { + continue; + } + + let senders_walker = sender_cursor.walk_range(tx_range)?; + for entry in senders_walker { + let (tx_num, sender) = entry?; + writer.append_transaction_sender(tx_num, &sender)?; + count += 1; + } + } + + writer.commit()?; + drop(provider); + + info!(target: "reth::cli", count, "TransactionSenders migrated"); + Ok(()) + } + + fn migrate_account_changesets( + &self, + tool: &DbTool, + tip: u64, + ) -> eyre::Result<()> { + info!(target: "reth::cli", "Migrating AccountChangeSets → static files"); + let provider = tool.provider_factory.provider()?; + let sf_provider = tool.provider_factory.static_file_provider(); + let mut writer = sf_provider.latest_writer(StaticFileSegment::AccountChangeSets)?; + + let mut cursor = provider.tx_ref().cursor_read::()?; + + let mut count = 0u64; + // Use a peekable walker so we can look ahead without consuming + let mut walker = cursor.walk(Some(0))?.peekable(); + + // Iterate ALL blocks from 0..=tip, appending empty changesets for blocks with no entries + for block in 0..=tip { + let mut entries = Vec::new(); + + // Collect all entries for this block + while let Some(Ok((block_number, _))) = walker.peek() { + if *block_number != block { + break; + } + let (_, entry) = walker.next().expect("peeked")?; + entries.push(entry); + } + + count += entries.len() as u64; + writer.append_account_changeset(entries, block)?; + } + + writer.commit()?; + drop(provider); + + info!(target: "reth::cli", count, "AccountChangeSets migrated"); + Ok(()) + } + + fn migrate_storage_changesets( + &self, + tool: &DbTool, + tip: u64, + ) -> eyre::Result<()> { + info!(target: "reth::cli", "Migrating StorageChangeSets → static files"); + let provider = tool.provider_factory.provider()?; + let sf_provider = tool.provider_factory.static_file_provider(); + let mut writer = sf_provider.latest_writer(StaticFileSegment::StorageChangeSets)?; + + let mut cursor = provider.tx_ref().cursor_read::()?; + + let mut count = 0u64; + let mut walker = cursor.walk(Some(Default::default()))?.peekable(); + + // Iterate ALL blocks from 0..=tip, appending empty changesets for blocks with no entries + for block in 0..=tip { + let mut entries = Vec::new(); + + // Collect all entries for this block + while let Some(Ok((key, _))) = walker.peek() { + if key.block_number() != block { + break; + } + let (key, entry) = walker.next().expect("peeked")?; + entries.push(StorageBeforeTx { + address: key.address(), + key: entry.key, + value: entry.value, + }); + } + + count += entries.len() as u64; + writer.append_storage_changeset(entries, block)?; + } + + writer.commit()?; + drop(provider); + + info!(target: "reth::cli", count, "StorageChangeSets migrated"); + Ok(()) + } + + fn migrate_receipts(&self, tool: &DbTool, tip: u64) -> eyre::Result<()> + where + N::Primitives: reth_primitives_traits::NodePrimitives< + Receipt: reth_db_api::table::Value + reth_codecs::Compact, + >, + { + let sf_provider = tool.provider_factory.static_file_provider(); + let existing = sf_provider.get_highest_static_file_block(StaticFileSegment::Receipts); + + if existing.is_some_and(|b| b >= tip) { + info!(target: "reth::cli", "Receipts already in static files, skipping"); + return Ok(()); + } + + info!(target: "reth::cli", "Migrating Receipts → static files"); + + let start_block = existing.map_or(0, |b| b + 1); + let block_range = start_block..=tip; + + // Use existing Segment implementation for receipts + let provider = tool.provider_factory.provider()?.disable_long_read_transaction_safety(); + + let segment = reth_static_file::segments::Receipts; + reth_static_file::segments::Segment::copy_to_static_files(&segment, provider, block_range)?; + + sf_provider.commit()?; + + info!(target: "reth::cli", "Receipts migrated"); + Ok(()) + } + + fn migrate_transaction_hash_numbers( + &self, + tool: &DbTool, + ) -> eyre::Result<()> { + info!(target: "reth::cli", "Migrating TransactionHashNumbers → RocksDB"); + let provider = tool.provider_factory.provider()?; + let rocksdb = tool.provider_factory.rocksdb_provider(); + + let mut cursor = provider.tx_ref().cursor_read::()?; + let mut batch = rocksdb.batch_with_auto_commit(); + + let mut count = 0u64; + let walker = cursor.walk(None)?; + for result in walker { + let (key, value) = result?; + batch.put::(key, &value)?; + count += 1; + if count.is_multiple_of(1_000_000) { + info!(target: "reth::cli", count, "TransactionHashNumbers progress"); + } + } + + batch.commit()?; + drop(provider); + + info!(target: "reth::cli", count, "TransactionHashNumbers migrated"); + Ok(()) + } + + fn migrate_accounts_history(&self, tool: &DbTool) -> eyre::Result<()> { + info!(target: "reth::cli", "Migrating AccountsHistory → RocksDB"); + let provider = tool.provider_factory.provider()?; + let rocksdb = tool.provider_factory.rocksdb_provider(); + + let mut cursor = provider.tx_ref().cursor_read::()?; + let mut batch = rocksdb.batch_with_auto_commit(); + + let mut count = 0u64; + let walker = cursor.walk(None)?; + for result in walker { + let (key, value) = result?; + batch.put::(key, &value)?; + count += 1; + if count.is_multiple_of(1_000_000) { + info!(target: "reth::cli", count, "AccountsHistory progress"); + } + } + + batch.commit()?; + drop(provider); + + info!(target: "reth::cli", count, "AccountsHistory migrated"); + Ok(()) + } + + fn migrate_storages_history(&self, tool: &DbTool) -> eyre::Result<()> { + info!(target: "reth::cli", "Migrating StoragesHistory → RocksDB"); + let provider = tool.provider_factory.provider()?; + let rocksdb = tool.provider_factory.rocksdb_provider(); + + let mut cursor = provider.tx_ref().cursor_read::()?; + let mut batch = rocksdb.batch_with_auto_commit(); + + let mut count = 0u64; + let walker = cursor.walk(None)?; + for result in walker { + let (key, value) = result?; + batch.put::(key, &value)?; + count += 1; + if count.is_multiple_of(1_000_000) { + info!(target: "reth::cli", count, "StoragesHistory progress"); + } + } + + batch.commit()?; + drop(provider); + + info!(target: "reth::cli", count, "StoragesHistory migrated"); + Ok(()) + } + + fn verify_hashed_state( + &self, + tool: &DbTool, + tip: u64, + ) -> eyre::Result<()> { + if tip == 0 { + info!(target: "reth::cli", "Empty chain, skipping hashed state verification"); + return Ok(()); + } + + info!(target: "reth::cli", "Verifying HashedAccounts/HashedStorages are populated"); + let provider = tool.provider_factory.provider()?; + + // Check AccountHashing + let account_hashing = provider + .get_stage_checkpoint(StageId::AccountHashing)? + .map(|c| c.block_number) + .unwrap_or(0); + + eyre::ensure!( + account_hashing >= tip, + "AccountHashing stage checkpoint ({account_hashing}) is behind execution tip ({tip}). \ + HashedAccounts may not be fully populated." + ); + + // Check StorageHashing + let storage_hashing = provider + .get_stage_checkpoint(StageId::StorageHashing)? + .map(|c| c.block_number) + .unwrap_or(0); + + eyre::ensure!( + storage_hashing >= tip, + "StorageHashing stage checkpoint ({storage_hashing}) is behind execution tip ({tip}). \ + HashedStorages may not be fully populated." + ); + + // Spot-check that HashedAccounts has at least one entry + let mut cursor = provider.tx_ref().cursor_read::()?; + eyre::ensure!( + cursor.first()?.is_some(), + "HashedAccounts table is empty but chain has state." + ); + + drop(provider); + info!(target: "reth::cli", "Hashed state verification passed"); + Ok(()) + } + + fn prune_migrated_tables(&self, tool: &DbTool) -> eyre::Result<()> { + info!(target: "reth::cli", "Pruning migrated MDBX tables"); + let db = tool.provider_factory.db_ref(); + + macro_rules! clear_table { + ($table:ty) => {{ + let tx = db.tx_mut()?; + tx.clear::<$table>()?; + tx.commit()?; + info!(target: "reth::cli", table = <$table as Table>::NAME, "Cleared"); + }}; + } + + clear_table!(tables::TransactionSenders); + clear_table!(tables::AccountChangeSets); + clear_table!(tables::StorageChangeSets); + clear_table!(tables::TransactionHashNumbers); + clear_table!(tables::AccountsHistory); + clear_table!(tables::StoragesHistory); + + info!(target: "reth::cli", "MDBX tables pruned. Consider running `mdbx_copy -c` to compact the database file."); + Ok(()) + } +} diff --git a/crates/cli/commands/src/db/mod.rs b/crates/cli/commands/src/db/mod.rs index bdfdb3b71eb..297423c26d1 100644 --- a/crates/cli/commands/src/db/mod.rs +++ b/crates/cli/commands/src/db/mod.rs @@ -16,6 +16,7 @@ mod copy; mod diff; mod get; mod list; +mod migrate_v2; mod prune_checkpoints; mod repair_trie; mod settings; @@ -77,6 +78,9 @@ pub enum Subcommands { AccountStorage(account_storage::Command), /// Gets account state and storage at a specific block State(state::Command), + /// Migrate storage layout from v1 (MDBX-only) to v2 (static files + RocksDB) + #[command(name = "migrate-v2")] + MigrateV2(migrate_v2::Command), } impl> Command { @@ -231,6 +235,11 @@ impl> Command command.execute(&tool)?; }); } + Subcommands::MigrateV2(command) => { + db_exec!(self.env, tool, N, AccessRights::RW, { + command.execute(&tool)?; + }); + } } Ok(()) From 0906f1b8d8581a88ba881869daa969c9657402e4 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 9 Apr 2026 14:38:02 +0000 Subject: [PATCH 02/11] feat(cli): prune plain state tables and auto-compact MDBX Adds PlainAccountState and PlainStorageState to the --prune-mdbx table list (superseded by HashedAccounts/HashedStorages in v2). Runs mdbx_env_copy with MDBX_CP_COMPACT after pruning to reclaim freed space, then swaps the compacted copy in after dropping the DB handle. Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019d725d-8f9f-740f-abae-cea809eeb511 --- crates/cli/commands/src/db/migrate_v2.rs | 84 ++++++++++++++++++++++-- crates/cli/commands/src/db/mod.rs | 16 ++++- 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/crates/cli/commands/src/db/migrate_v2.rs b/crates/cli/commands/src/db/migrate_v2.rs index 6892169978d..2a772a756df 100644 --- a/crates/cli/commands/src/db/migrate_v2.rs +++ b/crates/cli/commands/src/db/migrate_v2.rs @@ -12,7 +12,10 @@ //! Then updates `StorageSettings` to v2. use clap::Parser; -use reth_db::models::StorageBeforeTx; +use reth_db::{ + mdbx::{self, ffi}, + models::StorageBeforeTx, +}; use reth_db_api::{ cursor::DbCursorRO, database::Database, @@ -29,6 +32,7 @@ use reth_provider::{ use reth_stages_types::StageId; use reth_static_file_types::StaticFileSegment; use reth_storage_api::StageCheckpointReader; +use std::path::PathBuf; use tracing::info; /// `reth db migrate-v2` command @@ -41,7 +45,11 @@ pub struct Command { impl Command { /// Execute the migration. - pub fn execute(self, tool: &DbTool) -> eyre::Result<()> + /// + /// Returns `true` if MDBX tables were pruned and the database should be + /// compacted. The caller should run [`Self::compact_mdbx`] and + /// [`Self::swap_compacted_db`] after dropping the database handle. + pub fn execute(self, tool: &DbTool) -> eyre::Result where N::Primitives: reth_primitives_traits::NodePrimitives< Receipt: reth_db_api::table::Value + reth_codecs::Compact, @@ -55,7 +63,7 @@ impl Command { if current_settings.is_some_and(|s| s.is_v2()) { info!(target: "reth::cli", "Storage is already v2, nothing to do"); - return Ok(()); + return Ok(false); } let tip = @@ -131,6 +139,34 @@ impl Command { } info!(target: "reth::cli", "Migration complete!"); + Ok(self.prune_mdbx) + } + + /// Swaps the original MDBX database with a compacted copy. + /// + /// Must be called after the database handle has been dropped. + pub fn swap_compacted_db( + db_path: &std::path::Path, + compact_path: &std::path::Path, + ) -> eyre::Result<()> { + let backup_path = db_path.with_file_name("db_pre_compact"); + + info!(target: "reth::cli", ?db_path, ?compact_path, "Swapping compacted database"); + + // Rename original → backup + std::fs::rename(db_path, &backup_path)?; + + // Rename compacted → original + if let Err(e) = std::fs::rename(compact_path, db_path) { + // Restore backup on failure + let _ = std::fs::rename(&backup_path, db_path); + return Err(e.into()); + } + + // Remove backup + std::fs::remove_dir_all(&backup_path)?; + + info!(target: "reth::cli", "Database compaction swap complete"); Ok(()) } @@ -433,14 +469,54 @@ impl Command { }}; } + // Tables migrated to static files clear_table!(tables::TransactionSenders); clear_table!(tables::AccountChangeSets); clear_table!(tables::StorageChangeSets); + + // Tables migrated to RocksDB clear_table!(tables::TransactionHashNumbers); clear_table!(tables::AccountsHistory); clear_table!(tables::StoragesHistory); - info!(target: "reth::cli", "MDBX tables pruned. Consider running `mdbx_copy -c` to compact the database file."); + // Plain state tables superseded by hashed state in v2 + clear_table!(tables::PlainAccountState); + clear_table!(tables::PlainStorageState); + + info!(target: "reth::cli", "MDBX tables pruned"); Ok(()) } + + /// Creates a compacted copy of the MDBX database to `/../db_compact/`. + /// + /// Returns the path to the compacted copy. The caller must swap it with the + /// original after dropping the database handle. + pub fn compact_mdbx(db: &mdbx::DatabaseEnv) -> eyre::Result { + let db_path = db.path(); + let compact_path = db_path.with_file_name("db_compact"); + + reth_fs_util::create_dir_all(&compact_path)?; + + info!(target: "reth::cli", ?db_path, ?compact_path, "Compacting MDBX database"); + + let compact_dest = compact_path.join("mdbx.dat"); + let dest_cstr = std::ffi::CString::new( + compact_dest.to_str().ok_or_else(|| eyre::eyre!("compact path must be valid UTF-8"))?, + )?; + + let flags = ffi::MDBX_CP_COMPACT | ffi::MDBX_CP_FORCE_DYNAMIC_SIZE; + + let rc = db.with_raw_env_ptr(|env_ptr| unsafe { + ffi::mdbx_env_copy(env_ptr, dest_cstr.as_ptr(), flags) + }); + + if rc != 0 { + eyre::bail!("mdbx_env_copy failed with error code {rc}: {}", unsafe { + std::ffi::CStr::from_ptr(ffi::mdbx_strerror(rc)).to_string_lossy() + }); + } + + info!(target: "reth::cli", "MDBX compaction complete"); + Ok(compact_path) + } } diff --git a/crates/cli/commands/src/db/mod.rs b/crates/cli/commands/src/db/mod.rs index 297423c26d1..6ea76329a20 100644 --- a/crates/cli/commands/src/db/mod.rs +++ b/crates/cli/commands/src/db/mod.rs @@ -236,9 +236,23 @@ impl> Command }); } Subcommands::MigrateV2(command) => { + let needs_compact; db_exec!(self.env, tool, N, AccessRights::RW, { - command.execute(&tool)?; + needs_compact = command.execute(&tool)?; + + // Compaction must happen while we still hold the DB handle + // (mdbx_env_copy works on a live database). The compacted + // copy is written to `db_compact/` next to the original. + if needs_compact { + migrate_v2::Command::compact_mdbx(tool.provider_factory.db_ref())?; + } }); + + // After the DB handle is dropped, swap the compacted copy in. + if needs_compact { + let compact_path = db_path.with_file_name("db_compact"); + migrate_v2::Command::swap_compacted_db(&db_path, &compact_path)?; + } } } From 3299454fd5e91d7a94b781710da9885b6d30e0eb Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 9 Apr 2026 14:41:29 +0000 Subject: [PATCH 03/11] fix(cli): handle pruned nodes in migrate-v2 Static files may not start at block 0 on pruned nodes. Use cursor.first() to find the actual first available block, then get_writer(first_block, segment) instead of latest_writer(segment). Also call ensure_at_block(tip) for TransactionSenders to fill trailing empty blocks. Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019d725d-8f9f-740f-abae-cea809eeb511 --- crates/cli/commands/src/db/migrate_v2.rs | 56 +++++++++++++++++++----- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/crates/cli/commands/src/db/migrate_v2.rs b/crates/cli/commands/src/db/migrate_v2.rs index 2a772a756df..133f5495b52 100644 --- a/crates/cli/commands/src/db/migrate_v2.rs +++ b/crates/cli/commands/src/db/migrate_v2.rs @@ -178,13 +178,24 @@ impl Command { info!(target: "reth::cli", "Migrating TransactionSenders → static files"); let provider = tool.provider_factory.provider()?; let sf_provider = tool.provider_factory.static_file_provider(); - let mut writer = sf_provider.latest_writer(StaticFileSegment::TransactionSenders)?; let mut sender_cursor = provider.tx_ref().cursor_read::()?; let mut block_cursor = provider.tx_ref().cursor_read::()?; + // Find the first available block (may be non-zero on pruned nodes) + let first_block = match block_cursor.first()? { + Some((block, _)) => block, + None => { + info!(target: "reth::cli", "No BlockBodyIndices found, skipping TransactionSenders"); + return Ok(()); + } + }; + + let mut writer = + sf_provider.get_writer(first_block, StaticFileSegment::TransactionSenders)?; + let mut count = 0u64; - let block_walker = block_cursor.walk(Some(0))?; + let block_walker = block_cursor.walk(Some(first_block))?; for result in block_walker { let (block_number, body_indices) = result?; if block_number > tip { @@ -205,6 +216,8 @@ impl Command { } } + // Fill trailing empty blocks up to tip + writer.ensure_at_block(tip)?; writer.commit()?; drop(provider); @@ -220,19 +233,28 @@ impl Command { info!(target: "reth::cli", "Migrating AccountChangeSets → static files"); let provider = tool.provider_factory.provider()?; let sf_provider = tool.provider_factory.static_file_provider(); - let mut writer = sf_provider.latest_writer(StaticFileSegment::AccountChangeSets)?; let mut cursor = provider.tx_ref().cursor_read::()?; + // Find the first available block + let first_block = match cursor.first()? { + Some((block, _)) => block, + None => { + info!(target: "reth::cli", "No AccountChangeSets found, skipping"); + return Ok(()); + } + }; + + let mut writer = + sf_provider.get_writer(first_block, StaticFileSegment::AccountChangeSets)?; + let mut count = 0u64; - // Use a peekable walker so we can look ahead without consuming - let mut walker = cursor.walk(Some(0))?.peekable(); + let mut walker = cursor.walk(Some(first_block))?.peekable(); - // Iterate ALL blocks from 0..=tip, appending empty changesets for blocks with no entries - for block in 0..=tip { + // Iterate all blocks from first_block..=tip, including empty ones + for block in first_block..=tip { let mut entries = Vec::new(); - // Collect all entries for this block while let Some(Ok((block_number, _))) = walker.peek() { if *block_number != block { break; @@ -260,18 +282,28 @@ impl Command { info!(target: "reth::cli", "Migrating StorageChangeSets → static files"); let provider = tool.provider_factory.provider()?; let sf_provider = tool.provider_factory.static_file_provider(); - let mut writer = sf_provider.latest_writer(StaticFileSegment::StorageChangeSets)?; let mut cursor = provider.tx_ref().cursor_read::()?; + // Find the first available block + let first_block = match cursor.first()? { + Some((key, _)) => key.block_number(), + None => { + info!(target: "reth::cli", "No StorageChangeSets found, skipping"); + return Ok(()); + } + }; + + let mut writer = + sf_provider.get_writer(first_block, StaticFileSegment::StorageChangeSets)?; + let mut count = 0u64; let mut walker = cursor.walk(Some(Default::default()))?.peekable(); - // Iterate ALL blocks from 0..=tip, appending empty changesets for blocks with no entries - for block in 0..=tip { + // Iterate all blocks from first_block..=tip, including empty ones + for block in first_block..=tip { let mut entries = Vec::new(); - // Collect all entries for this block while let Some(Ok((key, _))) = walker.peek() { if key.block_number() != block { break; From 378b7377107bbb1004e52fa2e19e97d8a44d7555 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 9 Apr 2026 14:47:40 +0000 Subject: [PATCH 04/11] fix(cli): make prune+compact default, skip receipts with log filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove --prune-mdbx flag: pruning migrated tables (including PlainAccountState/PlainStorageState) and compaction are now always performed as part of the migration. - Skip receipt migration when receipts_log_filter pruning is enabled (receipts must stay in MDBX for log filter queries). - Remove Command fields — it's now a unit struct. Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019d725d-8f9f-740f-abae-cea809eeb511 --- crates/cli/commands/src/db/migrate_v2.rs | 35 ++++++++++++++---------- crates/cli/commands/src/db/mod.rs | 13 +++------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/crates/cli/commands/src/db/migrate_v2.rs b/crates/cli/commands/src/db/migrate_v2.rs index 133f5495b52..e990dcaa709 100644 --- a/crates/cli/commands/src/db/migrate_v2.rs +++ b/crates/cli/commands/src/db/migrate_v2.rs @@ -37,19 +37,16 @@ use tracing::info; /// `reth db migrate-v2` command #[derive(Debug, Parser)] -pub struct Command { - /// Prune migrated data from MDBX tables after successful migration. - #[arg(long)] - prune_mdbx: bool, -} +pub struct Command; impl Command { /// Execute the migration. /// - /// Returns `true` if MDBX tables were pruned and the database should be - /// compacted. The caller should run [`Self::compact_mdbx`] and - /// [`Self::swap_compacted_db`] after dropping the database handle. - pub fn execute(self, tool: &DbTool) -> eyre::Result + /// Migrates all v1 data to v2 layout, prunes the now-redundant MDBX tables + /// (including plain state), and compacts the database. The caller must run + /// [`Self::compact_mdbx`] while the DB handle is still open, then + /// [`Self::swap_compacted_db`] after dropping it. + pub fn execute(self, tool: &DbTool) -> eyre::Result<()> where N::Primitives: reth_primitives_traits::NodePrimitives< Receipt: reth_db_api::table::Value + reth_codecs::Compact, @@ -63,7 +60,7 @@ impl Command { if current_settings.is_some_and(|s| s.is_v2()) { info!(target: "reth::cli", "Storage is already v2, nothing to do"); - return Ok(false); + return Ok(()); } let tip = @@ -133,13 +130,11 @@ impl Command { provider_rw.commit()?; info!(target: "reth::cli", "Storage settings updated to v2"); - // === Phase 10: Optional MDBX pruning === - if self.prune_mdbx { - self.prune_migrated_tables(tool)?; - } + // === Phase 10: Prune migrated MDBX tables and plain state === + self.prune_migrated_tables(tool)?; info!(target: "reth::cli", "Migration complete!"); - Ok(self.prune_mdbx) + Ok(()) } /// Swaps the original MDBX database with a compacted copy. @@ -333,6 +328,16 @@ impl Command { Receipt: reth_db_api::table::Value + reth_codecs::Compact, >, { + // If receipt log filter pruning is enabled, receipts must stay in MDBX + // (v2 doesn't support static file receipts with log filter pruning yet). + let provider = tool.provider_factory.provider()?; + if !provider.prune_modes_ref().receipts_log_filter.is_empty() { + info!(target: "reth::cli", "Receipt log filter pruning is enabled, keeping receipts in MDBX"); + drop(provider); + return Ok(()); + } + drop(provider); + let sf_provider = tool.provider_factory.static_file_provider(); let existing = sf_provider.get_highest_static_file_block(StaticFileSegment::Receipts); diff --git a/crates/cli/commands/src/db/mod.rs b/crates/cli/commands/src/db/mod.rs index 6ea76329a20..993327a9afd 100644 --- a/crates/cli/commands/src/db/mod.rs +++ b/crates/cli/commands/src/db/mod.rs @@ -236,23 +236,18 @@ impl> Command }); } Subcommands::MigrateV2(command) => { - let needs_compact; db_exec!(self.env, tool, N, AccessRights::RW, { - needs_compact = command.execute(&tool)?; + command.execute(&tool)?; // Compaction must happen while we still hold the DB handle // (mdbx_env_copy works on a live database). The compacted // copy is written to `db_compact/` next to the original. - if needs_compact { - migrate_v2::Command::compact_mdbx(tool.provider_factory.db_ref())?; - } + migrate_v2::Command::compact_mdbx(tool.provider_factory.db_ref())?; }); // After the DB handle is dropped, swap the compacted copy in. - if needs_compact { - let compact_path = db_path.with_file_name("db_compact"); - migrate_v2::Command::swap_compacted_db(&db_path, &compact_path)?; - } + let compact_path = db_path.with_file_name("db_compact"); + migrate_v2::Command::swap_compacted_db(&db_path, &compact_path)?; } } From 9c7352b4b0a3ac0aaf59f94330f09db614888164 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 9 Apr 2026 14:54:22 +0000 Subject: [PATCH 05/11] fix(cli): handle non-zero start block for receipts migration On pruned nodes, BlockBodyIndices may not start at block 0. Use cursor.first() to find the first available block, same as the other segment migrations. Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019d725d-8f9f-740f-abae-cea809eeb511 --- crates/cli/commands/src/db/migrate_v2.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/cli/commands/src/db/migrate_v2.rs b/crates/cli/commands/src/db/migrate_v2.rs index e990dcaa709..1db883d18af 100644 --- a/crates/cli/commands/src/db/migrate_v2.rs +++ b/crates/cli/commands/src/db/migrate_v2.rs @@ -348,11 +348,19 @@ impl Command { info!(target: "reth::cli", "Migrating Receipts → static files"); - let start_block = existing.map_or(0, |b| b + 1); - let block_range = start_block..=tip; - - // Use existing Segment implementation for receipts + // Find the first block that has receipts data (may be non-zero on pruned nodes) let provider = tool.provider_factory.provider()?.disable_long_read_transaction_safety(); + let mut block_cursor = provider.tx_ref().cursor_read::()?; + let first_block = match block_cursor.first()? { + Some((block, _)) => block.max(existing.map_or(0, |b| b + 1)), + None => { + info!(target: "reth::cli", "No BlockBodyIndices found, skipping Receipts"); + return Ok(()); + } + }; + drop(block_cursor); + + let block_range = first_block..=tip; let segment = reth_static_file::segments::Receipts; reth_static_file::segments::Segment::copy_to_static_files(&segment, provider, block_range)?; From 84f4a6851bfb4b650e4fe5ca48c71c4a9b160a5c Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 9 Apr 2026 15:01:25 +0000 Subject: [PATCH 06/11] fix(cli): use prune checkpoints for start block, clear trie tables BlockBodyIndices is never pruned, so cursor.first() always returns block 0. Use PruneCheckpoints instead to find the first unpruned block per segment (SenderRecovery, AccountHistory, StorageHistory, Receipts). Also clear AccountsTrie and StoragesTrie tables and reset the MerkleExecute stage checkpoint to 0 so the trie is rebuilt on next startup. Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019d725d-8f9f-740f-abae-cea809eeb511 --- crates/cli/commands/src/db/migrate_v2.rs | 74 ++++++++++++------------ 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/crates/cli/commands/src/db/migrate_v2.rs b/crates/cli/commands/src/db/migrate_v2.rs index 1db883d18af..e954d4fe886 100644 --- a/crates/cli/commands/src/db/migrate_v2.rs +++ b/crates/cli/commands/src/db/migrate_v2.rs @@ -26,10 +26,11 @@ use reth_db_api::{ use reth_db_common::DbTool; use reth_provider::{ providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory, MetadataProvider, - MetadataWriter, RocksDBProviderFactory, StaticFileProviderFactory, StaticFileWriter, - StorageSettings, + MetadataWriter, PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointWriter, + StaticFileProviderFactory, StaticFileWriter, StorageSettings, }; -use reth_stages_types::StageId; +use reth_prune_types::PruneSegment; +use reth_stages_types::{StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; use reth_storage_api::StageCheckpointReader; use std::path::PathBuf; @@ -177,14 +178,12 @@ impl Command { let mut sender_cursor = provider.tx_ref().cursor_read::()?; let mut block_cursor = provider.tx_ref().cursor_read::()?; - // Find the first available block (may be non-zero on pruned nodes) - let first_block = match block_cursor.first()? { - Some((block, _)) => block, - None => { - info!(target: "reth::cli", "No BlockBodyIndices found, skipping TransactionSenders"); - return Ok(()); - } - }; + // Find the first unpruned block (SenderRecovery prune checkpoint tells us + // the highest pruned block; data starts at checkpoint + 1). + let first_block = provider + .get_prune_checkpoint(PruneSegment::SenderRecovery)? + .and_then(|cp| cp.block_number) + .map_or(0, |b| b + 1); let mut writer = sf_provider.get_writer(first_block, StaticFileSegment::TransactionSenders)?; @@ -231,14 +230,11 @@ impl Command { let mut cursor = provider.tx_ref().cursor_read::()?; - // Find the first available block - let first_block = match cursor.first()? { - Some((block, _)) => block, - None => { - info!(target: "reth::cli", "No AccountChangeSets found, skipping"); - return Ok(()); - } - }; + // Find the first unpruned block from AccountHistory prune checkpoint + let first_block = provider + .get_prune_checkpoint(PruneSegment::AccountHistory)? + .and_then(|cp| cp.block_number) + .map_or(0, |b| b + 1); let mut writer = sf_provider.get_writer(first_block, StaticFileSegment::AccountChangeSets)?; @@ -280,14 +276,11 @@ impl Command { let mut cursor = provider.tx_ref().cursor_read::()?; - // Find the first available block - let first_block = match cursor.first()? { - Some((key, _)) => key.block_number(), - None => { - info!(target: "reth::cli", "No StorageChangeSets found, skipping"); - return Ok(()); - } - }; + // Find the first unpruned block from StorageHistory prune checkpoint + let first_block = provider + .get_prune_checkpoint(PruneSegment::StorageHistory)? + .and_then(|cp| cp.block_number) + .map_or(0, |b| b + 1); let mut writer = sf_provider.get_writer(first_block, StaticFileSegment::StorageChangeSets)?; @@ -348,17 +341,13 @@ impl Command { info!(target: "reth::cli", "Migrating Receipts → static files"); - // Find the first block that has receipts data (may be non-zero on pruned nodes) + // Find the first unpruned block from Receipts prune checkpoint let provider = tool.provider_factory.provider()?.disable_long_read_transaction_safety(); - let mut block_cursor = provider.tx_ref().cursor_read::()?; - let first_block = match block_cursor.first()? { - Some((block, _)) => block.max(existing.map_or(0, |b| b + 1)), - None => { - info!(target: "reth::cli", "No BlockBodyIndices found, skipping Receipts"); - return Ok(()); - } - }; - drop(block_cursor); + let prune_start = provider + .get_prune_checkpoint(PruneSegment::Receipts)? + .and_then(|cp| cp.block_number) + .map_or(0, |b| b + 1); + let first_block = prune_start.max(existing.map_or(0, |b| b + 1)); let block_range = first_block..=tip; @@ -528,6 +517,17 @@ impl Command { clear_table!(tables::PlainAccountState); clear_table!(tables::PlainStorageState); + // Trie tables must be rebuilt (encoding changed between v1 and v2) + clear_table!(tables::AccountsTrie); + clear_table!(tables::StoragesTrie); + + // Reset Merkle checkpoint so the trie is rebuilt on next startup + let provider_rw = tool.provider_factory.database_provider_rw()?; + provider_rw.save_stage_checkpoint(StageId::MerkleExecute, StageCheckpoint::new(0))?; + provider_rw.save_stage_checkpoint_progress(StageId::MerkleExecute, vec![])?; + provider_rw.commit()?; + info!(target: "reth::cli", "MerkleExecute checkpoint reset to 0"); + info!(target: "reth::cli", "MDBX tables pruned"); Ok(()) } From df646b8f0626a44c623f7f971f70519a0fd5472e Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 9 Apr 2026 17:10:23 +0000 Subject: [PATCH 07/11] refactor(cli): only migrate changesets+receipts, let pipeline rebuild the rest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simplify migrate-v2 to only copy data that can't be recomputed: - AccountChangeSets → static files - StorageChangeSets → static files - Receipts → static files (unless log filter pruning) Then clear all recomputable tables and reset their stage checkpoints to 0. The pipeline will rebuild on next startup: - TransactionSenders (SenderRecovery) - TransactionHashNumbers (TransactionLookup) - AccountsHistory / StoragesHistory (IndexAccountHistory / IndexStorageHistory) - AccountsTrie / StoragesTrie (MerkleExecute) - PlainAccountState / PlainStorageState (no longer needed in v2) Also use disable_long_read_transaction_safety() on all migration providers to avoid tripping the long-tx watchdog. Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019d725d-8f9f-740f-abae-cea809eeb511 --- crates/cli/commands/src/db/migrate_v2.rs | 311 ++++------------------- 1 file changed, 52 insertions(+), 259 deletions(-) diff --git a/crates/cli/commands/src/db/migrate_v2.rs b/crates/cli/commands/src/db/migrate_v2.rs index e954d4fe886..380928393a7 100644 --- a/crates/cli/commands/src/db/migrate_v2.rs +++ b/crates/cli/commands/src/db/migrate_v2.rs @@ -1,15 +1,10 @@ //! `reth db migrate-v2` command for migrating v1 storage layout to v2. //! -//! Migrates data from MDBX-only (v1) storage layout to the hybrid v2 layout: -//! - TransactionSenders → static files -//! - AccountChangeSets → static files -//! - StorageChangeSets → static files -//! - Receipts → static files (if not already there) -//! - TransactionHashNumbers → RocksDB -//! - AccountsHistory → RocksDB -//! - StoragesHistory → RocksDB -//! -//! Then updates `StorageSettings` to v2. +//! Migrates data that cannot be recomputed (changesets + receipts) from MDBX to +//! static files, clears tables that *can* be recomputed (senders, indices, trie, +//! plain state), resets the corresponding stage checkpoints, and flips +//! `StorageSettings` to v2. The node will rebuild the cleared tables via the +//! normal pipeline on next startup. use clap::Parser; use reth_db::{ @@ -26,8 +21,8 @@ use reth_db_api::{ use reth_db_common::DbTool; use reth_provider::{ providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory, MetadataProvider, - MetadataWriter, PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointWriter, - StaticFileProviderFactory, StaticFileWriter, StorageSettings, + MetadataWriter, PruneCheckpointReader, StageCheckpointWriter, StaticFileProviderFactory, + StaticFileWriter, StorageSettings, }; use reth_prune_types::PruneSegment; use reth_stages_types::{StageCheckpoint, StageId}; @@ -43,10 +38,9 @@ pub struct Command; impl Command { /// Execute the migration. /// - /// Migrates all v1 data to v2 layout, prunes the now-redundant MDBX tables - /// (including plain state), and compacts the database. The caller must run - /// [`Self::compact_mdbx`] while the DB handle is still open, then - /// [`Self::swap_compacted_db`] after dropping it. + /// Only migrates changesets + receipts (data that cannot be recomputed), + /// then clears recomputable tables and resets their stage checkpoints. + /// The pipeline will rebuild senders, indices, trie, etc. on next startup. pub fn execute(self, tool: &DbTool) -> eyre::Result<()> where N::Primitives: reth_primitives_traits::NodePrimitives< @@ -72,11 +66,8 @@ impl Command { let sf_provider = tool.provider_factory.static_file_provider(); // Check that target static file segments are empty - for segment in [ - StaticFileSegment::TransactionSenders, - StaticFileSegment::AccountChangeSets, - StaticFileSegment::StorageChangeSets, - ] { + for segment in [StaticFileSegment::AccountChangeSets, StaticFileSegment::StorageChangeSets] + { if sf_provider.get_highest_static_file_block(segment).is_some() { eyre::bail!( "Static file segment {segment:?} already contains data. \ @@ -85,56 +76,31 @@ impl Command { } } - // Check that RocksDB tables are empty - let rocksdb = tool.provider_factory.rocksdb_provider(); - if rocksdb.first::()?.is_some() { - eyre::bail!("RocksDB TransactionHashNumbers already contains data"); - } - if rocksdb.first::()?.is_some() { - eyre::bail!("RocksDB AccountsHistory already contains data"); - } - if rocksdb.first::()?.is_some() { - eyre::bail!("RocksDB StoragesHistory already contains data"); - } - drop(provider); info!(target: "reth::cli", "Preflight checks passed"); - // === Phase 1: TransactionSenders → static files === - self.migrate_transaction_senders(tool, tip)?; - - // === Phase 2: AccountChangeSets → static files === + // === Phase 1: AccountChangeSets → static files === self.migrate_account_changesets(tool, tip)?; - // === Phase 3: StorageChangeSets → static files === + // === Phase 2: StorageChangeSets → static files === self.migrate_storage_changesets(tool, tip)?; - // === Phase 4: Receipts → static files === + // === Phase 3: Receipts → static files === self.migrate_receipts::(tool, tip)?; - // === Phase 5: TransactionHashNumbers → RocksDB === - self.migrate_transaction_hash_numbers(tool)?; - - // === Phase 6: AccountsHistory → RocksDB === - self.migrate_accounts_history(tool)?; - - // === Phase 7: StoragesHistory → RocksDB === - self.migrate_storages_history(tool)?; - - // === Phase 8: Verify hashed state === - self.verify_hashed_state(tool, tip)?; - - // === Phase 9: Update metadata to v2 === + // === Phase 4: Update metadata to v2 === info!(target: "reth::cli", "Writing StorageSettings v2 metadata"); - let provider_rw = tool.provider_factory.database_provider_rw()?; - provider_rw.write_storage_settings(StorageSettings::v2())?; - provider_rw.commit()?; + { + let provider_rw = tool.provider_factory.database_provider_rw()?; + provider_rw.write_storage_settings(StorageSettings::v2())?; + provider_rw.commit()?; + } info!(target: "reth::cli", "Storage settings updated to v2"); - // === Phase 10: Prune migrated MDBX tables and plain state === - self.prune_migrated_tables(tool)?; + // === Phase 5: Clear recomputable tables and reset stage checkpoints === + self.clear_recomputable_tables(tool)?; - info!(target: "reth::cli", "Migration complete!"); + info!(target: "reth::cli", "Migration complete! Start the node to rebuild indices, senders, and trie."); Ok(()) } @@ -149,83 +115,26 @@ impl Command { info!(target: "reth::cli", ?db_path, ?compact_path, "Swapping compacted database"); - // Rename original → backup std::fs::rename(db_path, &backup_path)?; - // Rename compacted → original if let Err(e) = std::fs::rename(compact_path, db_path) { - // Restore backup on failure let _ = std::fs::rename(&backup_path, db_path); return Err(e.into()); } - // Remove backup std::fs::remove_dir_all(&backup_path)?; info!(target: "reth::cli", "Database compaction swap complete"); Ok(()) } - fn migrate_transaction_senders( - &self, - tool: &DbTool, - tip: u64, - ) -> eyre::Result<()> { - info!(target: "reth::cli", "Migrating TransactionSenders → static files"); - let provider = tool.provider_factory.provider()?; - let sf_provider = tool.provider_factory.static_file_provider(); - - let mut sender_cursor = provider.tx_ref().cursor_read::()?; - let mut block_cursor = provider.tx_ref().cursor_read::()?; - - // Find the first unpruned block (SenderRecovery prune checkpoint tells us - // the highest pruned block; data starts at checkpoint + 1). - let first_block = provider - .get_prune_checkpoint(PruneSegment::SenderRecovery)? - .and_then(|cp| cp.block_number) - .map_or(0, |b| b + 1); - - let mut writer = - sf_provider.get_writer(first_block, StaticFileSegment::TransactionSenders)?; - - let mut count = 0u64; - let block_walker = block_cursor.walk(Some(first_block))?; - for result in block_walker { - let (block_number, body_indices) = result?; - if block_number > tip { - break; - } - writer.increment_block(block_number)?; - - let tx_range = body_indices.tx_num_range(); - if tx_range.is_empty() { - continue; - } - - let senders_walker = sender_cursor.walk_range(tx_range)?; - for entry in senders_walker { - let (tx_num, sender) = entry?; - writer.append_transaction_sender(tx_num, &sender)?; - count += 1; - } - } - - // Fill trailing empty blocks up to tip - writer.ensure_at_block(tip)?; - writer.commit()?; - drop(provider); - - info!(target: "reth::cli", count, "TransactionSenders migrated"); - Ok(()) - } - fn migrate_account_changesets( &self, tool: &DbTool, tip: u64, ) -> eyre::Result<()> { info!(target: "reth::cli", "Migrating AccountChangeSets → static files"); - let provider = tool.provider_factory.provider()?; + let provider = tool.provider_factory.provider()?.disable_long_read_transaction_safety(); let sf_provider = tool.provider_factory.static_file_provider(); let mut cursor = provider.tx_ref().cursor_read::()?; @@ -242,7 +151,6 @@ impl Command { let mut count = 0u64; let mut walker = cursor.walk(Some(first_block))?.peekable(); - // Iterate all blocks from first_block..=tip, including empty ones for block in first_block..=tip { let mut entries = Vec::new(); @@ -271,7 +179,7 @@ impl Command { tip: u64, ) -> eyre::Result<()> { info!(target: "reth::cli", "Migrating StorageChangeSets → static files"); - let provider = tool.provider_factory.provider()?; + let provider = tool.provider_factory.provider()?.disable_long_read_transaction_safety(); let sf_provider = tool.provider_factory.static_file_provider(); let mut cursor = provider.tx_ref().cursor_read::()?; @@ -288,7 +196,6 @@ impl Command { let mut count = 0u64; let mut walker = cursor.walk(Some(Default::default()))?.peekable(); - // Iterate all blocks from first_block..=tip, including empty ones for block in first_block..=tip { let mut entries = Vec::new(); @@ -322,7 +229,6 @@ impl Command { >, { // If receipt log filter pruning is enabled, receipts must stay in MDBX - // (v2 doesn't support static file receipts with log filter pruning yet). let provider = tool.provider_factory.provider()?; if !provider.prune_modes_ref().receipts_log_filter.is_empty() { info!(target: "reth::cli", "Receipt log filter pruning is enabled, keeping receipts in MDBX"); @@ -341,7 +247,6 @@ impl Command { info!(target: "reth::cli", "Migrating Receipts → static files"); - // Find the first unpruned block from Receipts prune checkpoint let provider = tool.provider_factory.provider()?.disable_long_read_transaction_safety(); let prune_start = provider .get_prune_checkpoint(PruneSegment::Receipts)? @@ -360,138 +265,13 @@ impl Command { Ok(()) } - fn migrate_transaction_hash_numbers( + /// Clears tables that can be recomputed by the pipeline and resets their + /// stage checkpoints. The node will rebuild them on next startup. + fn clear_recomputable_tables( &self, tool: &DbTool, ) -> eyre::Result<()> { - info!(target: "reth::cli", "Migrating TransactionHashNumbers → RocksDB"); - let provider = tool.provider_factory.provider()?; - let rocksdb = tool.provider_factory.rocksdb_provider(); - - let mut cursor = provider.tx_ref().cursor_read::()?; - let mut batch = rocksdb.batch_with_auto_commit(); - - let mut count = 0u64; - let walker = cursor.walk(None)?; - for result in walker { - let (key, value) = result?; - batch.put::(key, &value)?; - count += 1; - if count.is_multiple_of(1_000_000) { - info!(target: "reth::cli", count, "TransactionHashNumbers progress"); - } - } - - batch.commit()?; - drop(provider); - - info!(target: "reth::cli", count, "TransactionHashNumbers migrated"); - Ok(()) - } - - fn migrate_accounts_history(&self, tool: &DbTool) -> eyre::Result<()> { - info!(target: "reth::cli", "Migrating AccountsHistory → RocksDB"); - let provider = tool.provider_factory.provider()?; - let rocksdb = tool.provider_factory.rocksdb_provider(); - - let mut cursor = provider.tx_ref().cursor_read::()?; - let mut batch = rocksdb.batch_with_auto_commit(); - - let mut count = 0u64; - let walker = cursor.walk(None)?; - for result in walker { - let (key, value) = result?; - batch.put::(key, &value)?; - count += 1; - if count.is_multiple_of(1_000_000) { - info!(target: "reth::cli", count, "AccountsHistory progress"); - } - } - - batch.commit()?; - drop(provider); - - info!(target: "reth::cli", count, "AccountsHistory migrated"); - Ok(()) - } - - fn migrate_storages_history(&self, tool: &DbTool) -> eyre::Result<()> { - info!(target: "reth::cli", "Migrating StoragesHistory → RocksDB"); - let provider = tool.provider_factory.provider()?; - let rocksdb = tool.provider_factory.rocksdb_provider(); - - let mut cursor = provider.tx_ref().cursor_read::()?; - let mut batch = rocksdb.batch_with_auto_commit(); - - let mut count = 0u64; - let walker = cursor.walk(None)?; - for result in walker { - let (key, value) = result?; - batch.put::(key, &value)?; - count += 1; - if count.is_multiple_of(1_000_000) { - info!(target: "reth::cli", count, "StoragesHistory progress"); - } - } - - batch.commit()?; - drop(provider); - - info!(target: "reth::cli", count, "StoragesHistory migrated"); - Ok(()) - } - - fn verify_hashed_state( - &self, - tool: &DbTool, - tip: u64, - ) -> eyre::Result<()> { - if tip == 0 { - info!(target: "reth::cli", "Empty chain, skipping hashed state verification"); - return Ok(()); - } - - info!(target: "reth::cli", "Verifying HashedAccounts/HashedStorages are populated"); - let provider = tool.provider_factory.provider()?; - - // Check AccountHashing - let account_hashing = provider - .get_stage_checkpoint(StageId::AccountHashing)? - .map(|c| c.block_number) - .unwrap_or(0); - - eyre::ensure!( - account_hashing >= tip, - "AccountHashing stage checkpoint ({account_hashing}) is behind execution tip ({tip}). \ - HashedAccounts may not be fully populated." - ); - - // Check StorageHashing - let storage_hashing = provider - .get_stage_checkpoint(StageId::StorageHashing)? - .map(|c| c.block_number) - .unwrap_or(0); - - eyre::ensure!( - storage_hashing >= tip, - "StorageHashing stage checkpoint ({storage_hashing}) is behind execution tip ({tip}). \ - HashedStorages may not be fully populated." - ); - - // Spot-check that HashedAccounts has at least one entry - let mut cursor = provider.tx_ref().cursor_read::()?; - eyre::ensure!( - cursor.first()?.is_some(), - "HashedAccounts table is empty but chain has state." - ); - - drop(provider); - info!(target: "reth::cli", "Hashed state verification passed"); - Ok(()) - } - - fn prune_migrated_tables(&self, tool: &DbTool) -> eyre::Result<()> { - info!(target: "reth::cli", "Pruning migrated MDBX tables"); + info!(target: "reth::cli", "Clearing recomputable MDBX tables"); let db = tool.provider_factory.db_ref(); macro_rules! clear_table { @@ -503,32 +283,45 @@ impl Command { }}; } - // Tables migrated to static files - clear_table!(tables::TransactionSenders); + // Migrated changeset tables (now in static files) clear_table!(tables::AccountChangeSets); clear_table!(tables::StorageChangeSets); - // Tables migrated to RocksDB + // Senders — will be recomputed by SenderRecovery stage + clear_table!(tables::TransactionSenders); + + // Indices — will be recomputed by TransactionLookup / IndexAccountHistory / + // IndexStorageHistory clear_table!(tables::TransactionHashNumbers); clear_table!(tables::AccountsHistory); clear_table!(tables::StoragesHistory); - // Plain state tables superseded by hashed state in v2 + // Plain state — superseded by hashed state in v2 clear_table!(tables::PlainAccountState); clear_table!(tables::PlainStorageState); - // Trie tables must be rebuilt (encoding changed between v1 and v2) + // Trie — will be rebuilt by MerkleExecute clear_table!(tables::AccountsTrie); clear_table!(tables::StoragesTrie); - // Reset Merkle checkpoint so the trie is rebuilt on next startup + // Reset stage checkpoints so the pipeline rebuilds everything + info!(target: "reth::cli", "Resetting stage checkpoints"); let provider_rw = tool.provider_factory.database_provider_rw()?; - provider_rw.save_stage_checkpoint(StageId::MerkleExecute, StageCheckpoint::new(0))?; + for stage in [ + StageId::SenderRecovery, + StageId::TransactionLookup, + StageId::IndexAccountHistory, + StageId::IndexStorageHistory, + StageId::MerkleExecute, + StageId::MerkleUnwind, + ] { + provider_rw.save_stage_checkpoint(stage, StageCheckpoint::new(0))?; + info!(target: "reth::cli", %stage, "Checkpoint reset to 0"); + } provider_rw.save_stage_checkpoint_progress(StageId::MerkleExecute, vec![])?; provider_rw.commit()?; - info!(target: "reth::cli", "MerkleExecute checkpoint reset to 0"); - info!(target: "reth::cli", "MDBX tables pruned"); + info!(target: "reth::cli", "Recomputable tables cleared and checkpoints reset"); Ok(()) } From 1164899f4b044fd0d0909a51947f1f54694b6243 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 9 Apr 2026 17:16:46 +0000 Subject: [PATCH 08/11] feat(cli): run pipeline after migration to rebuild cleared tables Build a DefaultStages pipeline with noop downloaders/consensus/evm and max_block=tip, then run it after migration. Stages already at tip (Headers, Bodies, Execution) will no-op; stages with reset checkpoints (SenderRecovery, TransactionLookup, IndexAccountHistory, IndexStorageHistory, MerkleExecute) will execute and rebuild their tables. Compact MDBX after the pipeline finishes. Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019d725d-8f9f-740f-abae-cea809eeb511 --- crates/cli/commands/src/db/mod.rs | 73 +++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 9 deletions(-) diff --git a/crates/cli/commands/src/db/mod.rs b/crates/cli/commands/src/db/mod.rs index 993327a9afd..092549bf9ce 100644 --- a/crates/cli/commands/src/db/mod.rs +++ b/crates/cli/commands/src/db/mod.rs @@ -1,14 +1,28 @@ use crate::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs}; +use alloy_primitives::B256; use clap::{Parser, Subcommand}; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_cli::chainspec::ChainSpecParser; use reth_cli_runner::CliContext; -use reth_db::version::{get_db_version, DatabaseVersionError, DB_VERSION}; +use reth_consensus::noop::NoopConsensus; +use reth_db::{ + version::{get_db_version, DatabaseVersionError, DB_VERSION}, + DatabaseEnv, +}; use reth_db_common::DbTool; +use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader}; +use reth_evm::noop::NoopEvmConfig; +use reth_node_builder::NodeTypesWithDBAdapter; +use reth_provider::StageCheckpointReader; +use reth_stages::{sets::DefaultStages, Pipeline}; +use reth_stages_types::StageId; +use reth_static_file::StaticFileProducer; use std::{ io::{self, Write}, sync::Arc, }; +use tokio::sync::watch; +use tracing::info; mod account_storage; mod checksum; mod clear; @@ -236,16 +250,57 @@ impl> Command }); } Subcommands::MigrateV2(command) => { - db_exec!(self.env, tool, N, AccessRights::RW, { - command.execute(&tool)?; + let Environment { provider_factory, config, .. } = + self.env.init::(AccessRights::RW, ctx.task_executor.clone())?; - // Compaction must happen while we still hold the DB handle - // (mdbx_env_copy works on a live database). The compacted - // copy is written to `db_compact/` next to the original. - migrate_v2::Command::compact_mdbx(tool.provider_factory.db_ref())?; - }); + let tool = DbTool::new(provider_factory.clone())?; + command.execute(&tool)?; + + // Run pipeline to rebuild senders, indices, and trie + info!(target: "reth::cli", "Running pipeline to rebuild cleared tables"); + + let tip = provider_factory + .provider()? + .get_stage_checkpoint(StageId::Execution)? + .map(|c| c.block_number) + .unwrap_or(0); + + let (_tip_tx, tip_rx) = watch::channel(B256::ZERO); + + let mut pipeline = Pipeline::>::builder() + .with_max_block(tip) + .add_stages(DefaultStages::new( + provider_factory.clone(), + tip_rx, + Arc::new(NoopConsensus::default()), + NoopHeaderDownloader::default(), + NoopBodiesDownloader::default(), + NoopEvmConfig::::default(), + config.stages.clone(), + config.prune.segments.clone(), + None, + )) + .build( + provider_factory.clone(), + StaticFileProducer::new( + provider_factory.clone(), + config.prune.segments.clone(), + ), + ); + + pipeline.run().await?; + + info!(target: "reth::cli", "Pipeline finished, compacting MDBX"); + + // Compact MDBX while DB handle is still open + migrate_v2::Command::compact_mdbx(tool.provider_factory.db_ref())?; + + // Drop everything to release the DB handle + drop(pipeline); + drop(tool); + drop(provider_factory); - // After the DB handle is dropped, swap the compacted copy in. + // Swap compacted copy in let compact_path = db_path.with_file_name("db_compact"); migrate_v2::Command::swap_compacted_db(&db_path, &compact_path)?; } From d2327cb14f4fc42b3ab7afd37dd1c9e6346ee620 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 9 Apr 2026 17:34:21 +0000 Subject: [PATCH 09/11] refactor(cli): encapsulate migration in subcommand, compact before pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move all logic (pipeline + compaction) into migrate_v2::Command. mod.rs is now a thin wrapper: init env → execute → reopen → pipeline. Compaction now runs BEFORE the pipeline so the pipeline operates on the smaller compacted database. Flow: migrate data → flip v2 → clear tables → compact MDBX → swap → reopen → run pipeline to rebuild. Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019d725d-8f9f-740f-abae-cea809eeb511 --- crates/cli/commands/src/db/migrate_v2.rs | 222 ++++++++++++++--------- crates/cli/commands/src/db/mod.rs | 70 +------ 2 files changed, 147 insertions(+), 145 deletions(-) diff --git a/crates/cli/commands/src/db/migrate_v2.rs b/crates/cli/commands/src/db/migrate_v2.rs index 380928393a7..9c65cbcaf74 100644 --- a/crates/cli/commands/src/db/migrate_v2.rs +++ b/crates/cli/commands/src/db/migrate_v2.rs @@ -1,15 +1,18 @@ //! `reth db migrate-v2` command for migrating v1 storage layout to v2. //! //! Migrates data that cannot be recomputed (changesets + receipts) from MDBX to -//! static files, clears tables that *can* be recomputed (senders, indices, trie, -//! plain state), resets the corresponding stage checkpoints, and flips -//! `StorageSettings` to v2. The node will rebuild the cleared tables via the -//! normal pipeline on next startup. +//! static files, clears recomputable tables (senders, indices, trie, plain +//! state), compacts MDBX, then runs the pipeline to rebuild them. +use crate::common::CliNodeTypes; +use alloy_primitives::B256; use clap::Parser; +use reth_config::Config; +use reth_consensus::noop::NoopConsensus; use reth_db::{ mdbx::{self, ffi}, models::StorageBeforeTx, + DatabaseEnv, }; use reth_db_api::{ cursor::DbCursorRO, @@ -18,17 +21,22 @@ use reth_db_api::{ tables, transaction::{DbTx, DbTxMut}, }; -use reth_db_common::DbTool; +use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader}; +use reth_evm::noop::NoopEvmConfig; +use reth_node_builder::NodeTypesWithDBAdapter; use reth_provider::{ providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory, MetadataProvider, - MetadataWriter, PruneCheckpointReader, StageCheckpointWriter, StaticFileProviderFactory, - StaticFileWriter, StorageSettings, + MetadataWriter, ProviderFactory, PruneCheckpointReader, StageCheckpointWriter, + StaticFileProviderFactory, StaticFileWriter, StorageSettings, }; use reth_prune_types::PruneSegment; +use reth_stages::{sets::DefaultStages, Pipeline}; use reth_stages_types::{StageCheckpoint, StageId}; +use reth_static_file::StaticFileProducer; use reth_static_file_types::StaticFileSegment; use reth_storage_api::StageCheckpointReader; -use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::watch; use tracing::info; /// `reth db migrate-v2` command @@ -36,12 +44,17 @@ use tracing::info; pub struct Command; impl Command { - /// Execute the migration. + /// Execute the full v1 → v2 migration: /// - /// Only migrates changesets + receipts (data that cannot be recomputed), - /// then clears recomputable tables and resets their stage checkpoints. - /// The pipeline will rebuild senders, indices, trie, etc. on next startup. - pub fn execute(self, tool: &DbTool) -> eyre::Result<()> + /// 1. Migrate changesets + receipts to static files + /// 2. Flip `StorageSettings` to v2 + /// 3. Clear recomputable MDBX tables + reset stage checkpoints + /// 4. Compact MDBX + /// 5. Run pipeline to rebuild senders, indices, and trie + pub async fn execute( + self, + provider_factory: ProviderFactory>, + ) -> eyre::Result<()> where N::Primitives: reth_primitives_traits::NodePrimitives< Receipt: reth_db_api::table::Value + reth_codecs::Compact, @@ -50,7 +63,7 @@ impl Command { // === Phase 0: Preflight === info!(target: "reth::cli", "Starting v1 → v2 storage migration"); - let provider = tool.provider_factory.provider()?; + let provider = provider_factory.provider()?; let current_settings = provider.storage_settings()?; if current_settings.is_some_and(|s| s.is_v2()) { @@ -63,9 +76,8 @@ impl Command { info!(target: "reth::cli", tip, "Chain tip block number"); - let sf_provider = tool.provider_factory.static_file_provider(); + let sf_provider = provider_factory.static_file_provider(); - // Check that target static file segments are empty for segment in [StaticFileSegment::AccountChangeSets, StaticFileSegment::StorageChangeSets] { if sf_provider.get_highest_static_file_block(segment).is_some() { @@ -79,67 +91,99 @@ impl Command { drop(provider); info!(target: "reth::cli", "Preflight checks passed"); - // === Phase 1: AccountChangeSets → static files === - self.migrate_account_changesets(tool, tip)?; + // === Phase 1: Migrate changesets → static files === + Self::migrate_account_changesets(&provider_factory, tip)?; + Self::migrate_storage_changesets(&provider_factory, tip)?; - // === Phase 2: StorageChangeSets → static files === - self.migrate_storage_changesets(tool, tip)?; + // === Phase 2: Migrate receipts → static files === + Self::migrate_receipts::>(&provider_factory, tip)?; - // === Phase 3: Receipts → static files === - self.migrate_receipts::(tool, tip)?; - - // === Phase 4: Update metadata to v2 === + // === Phase 3: Flip metadata to v2 === info!(target: "reth::cli", "Writing StorageSettings v2 metadata"); { - let provider_rw = tool.provider_factory.database_provider_rw()?; + let provider_rw = provider_factory.database_provider_rw()?; provider_rw.write_storage_settings(StorageSettings::v2())?; provider_rw.commit()?; } info!(target: "reth::cli", "Storage settings updated to v2"); - // === Phase 5: Clear recomputable tables and reset stage checkpoints === - self.clear_recomputable_tables(tool)?; - - info!(target: "reth::cli", "Migration complete! Start the node to rebuild indices, senders, and trie."); - Ok(()) - } - - /// Swaps the original MDBX database with a compacted copy. - /// - /// Must be called after the database handle has been dropped. - pub fn swap_compacted_db( - db_path: &std::path::Path, - compact_path: &std::path::Path, - ) -> eyre::Result<()> { - let backup_path = db_path.with_file_name("db_pre_compact"); + // === Phase 4: Clear recomputable tables === + Self::clear_recomputable_tables(&provider_factory)?; - info!(target: "reth::cli", ?db_path, ?compact_path, "Swapping compacted database"); + // === Phase 5: Compact MDBX (before pipeline, so it runs on a smaller DB) === + let db_path = provider_factory.db_ref().path(); + Self::compact_mdbx(provider_factory.db_ref())?; - std::fs::rename(db_path, &backup_path)?; + // Drop to release DB handle for swap + drop(provider_factory); - if let Err(e) = std::fs::rename(compact_path, db_path) { - let _ = std::fs::rename(&backup_path, db_path); - return Err(e.into()); - } + let compact_path = db_path.with_file_name("db_compact"); + Self::swap_compacted_db(&db_path, &compact_path)?; - std::fs::remove_dir_all(&backup_path)?; + // === Phase 6: Reopen DB and run pipeline === + // The caller will reopen the environment and run the pipeline. + // We return here — the pipeline step is handled in mod.rs after + // reopening the database with the compacted copy. + info!(target: "reth::cli", "Migration data phases complete"); + Ok(()) + } - info!(target: "reth::cli", "Database compaction swap complete"); + /// Builds and runs the pipeline to rebuild cleared tables. + /// + /// Must be called after the database has been compacted and reopened. + pub async fn run_pipeline( + provider_factory: ProviderFactory>, + config: &Config, + ) -> eyre::Result<()> + where + N::Primitives: reth_primitives_traits::NodePrimitives< + Receipt: reth_db_api::table::Value + reth_codecs::Compact, + >, + { + let tip = provider_factory + .provider()? + .get_stage_checkpoint(StageId::Execution)? + .map(|c| c.block_number) + .unwrap_or(0); + + info!(target: "reth::cli", tip, "Running pipeline to rebuild tables"); + + let (_tip_tx, tip_rx) = watch::channel(B256::ZERO); + + let mut pipeline = Pipeline::>::builder() + .with_max_block(tip) + .add_stages(DefaultStages::new( + provider_factory.clone(), + tip_rx, + Arc::new(NoopConsensus::default()), + NoopHeaderDownloader::default(), + NoopBodiesDownloader::default(), + NoopEvmConfig::::default(), + config.stages.clone(), + config.prune.segments.clone(), + None, + )) + .build( + provider_factory.clone(), + StaticFileProducer::new(provider_factory, config.prune.segments.clone()), + ); + + pipeline.run().await?; + + info!(target: "reth::cli", "Pipeline finished"); Ok(()) } fn migrate_account_changesets( - &self, - tool: &DbTool, + factory: &ProviderFactory, tip: u64, ) -> eyre::Result<()> { info!(target: "reth::cli", "Migrating AccountChangeSets → static files"); - let provider = tool.provider_factory.provider()?.disable_long_read_transaction_safety(); - let sf_provider = tool.provider_factory.static_file_provider(); + let provider = factory.provider()?.disable_long_read_transaction_safety(); + let sf_provider = factory.static_file_provider(); let mut cursor = provider.tx_ref().cursor_read::()?; - // Find the first unpruned block from AccountHistory prune checkpoint let first_block = provider .get_prune_checkpoint(PruneSegment::AccountHistory)? .and_then(|cp| cp.block_number) @@ -167,24 +211,21 @@ impl Command { } writer.commit()?; - drop(provider); info!(target: "reth::cli", count, "AccountChangeSets migrated"); Ok(()) } fn migrate_storage_changesets( - &self, - tool: &DbTool, + factory: &ProviderFactory, tip: u64, ) -> eyre::Result<()> { info!(target: "reth::cli", "Migrating StorageChangeSets → static files"); - let provider = tool.provider_factory.provider()?.disable_long_read_transaction_safety(); - let sf_provider = tool.provider_factory.static_file_provider(); + let provider = factory.provider()?.disable_long_read_transaction_safety(); + let sf_provider = factory.static_file_provider(); let mut cursor = provider.tx_ref().cursor_read::()?; - // Find the first unpruned block from StorageHistory prune checkpoint let first_block = provider .get_prune_checkpoint(PruneSegment::StorageHistory)? .and_then(|cp| cp.block_number) @@ -216,28 +257,28 @@ impl Command { } writer.commit()?; - drop(provider); info!(target: "reth::cli", count, "StorageChangeSets migrated"); Ok(()) } - fn migrate_receipts(&self, tool: &DbTool, tip: u64) -> eyre::Result<()> + fn migrate_receipts( + factory: &ProviderFactory, + tip: u64, + ) -> eyre::Result<()> where N::Primitives: reth_primitives_traits::NodePrimitives< Receipt: reth_db_api::table::Value + reth_codecs::Compact, >, { - // If receipt log filter pruning is enabled, receipts must stay in MDBX - let provider = tool.provider_factory.provider()?; + let provider = factory.provider()?; if !provider.prune_modes_ref().receipts_log_filter.is_empty() { info!(target: "reth::cli", "Receipt log filter pruning is enabled, keeping receipts in MDBX"); - drop(provider); return Ok(()); } drop(provider); - let sf_provider = tool.provider_factory.static_file_provider(); + let sf_provider = factory.static_file_provider(); let existing = sf_provider.get_highest_static_file_block(StaticFileSegment::Receipts); if existing.is_some_and(|b| b >= tip) { @@ -247,7 +288,7 @@ impl Command { info!(target: "reth::cli", "Migrating Receipts → static files"); - let provider = tool.provider_factory.provider()?.disable_long_read_transaction_safety(); + let provider = factory.provider()?.disable_long_read_transaction_safety(); let prune_start = provider .get_prune_checkpoint(PruneSegment::Receipts)? .and_then(|cp| cp.block_number) @@ -266,13 +307,12 @@ impl Command { } /// Clears tables that can be recomputed by the pipeline and resets their - /// stage checkpoints. The node will rebuild them on next startup. + /// stage checkpoints. fn clear_recomputable_tables( - &self, - tool: &DbTool, + factory: &ProviderFactory, ) -> eyre::Result<()> { info!(target: "reth::cli", "Clearing recomputable MDBX tables"); - let db = tool.provider_factory.db_ref(); + let db = factory.db_ref(); macro_rules! clear_table { ($table:ty) => {{ @@ -287,11 +327,10 @@ impl Command { clear_table!(tables::AccountChangeSets); clear_table!(tables::StorageChangeSets); - // Senders — will be recomputed by SenderRecovery stage + // Senders — rebuilt by SenderRecovery clear_table!(tables::TransactionSenders); - // Indices — will be recomputed by TransactionLookup / IndexAccountHistory / - // IndexStorageHistory + // Indices — rebuilt by TransactionLookup / IndexAccountHistory / IndexStorageHistory clear_table!(tables::TransactionHashNumbers); clear_table!(tables::AccountsHistory); clear_table!(tables::StoragesHistory); @@ -300,13 +339,13 @@ impl Command { clear_table!(tables::PlainAccountState); clear_table!(tables::PlainStorageState); - // Trie — will be rebuilt by MerkleExecute + // Trie — rebuilt by MerkleExecute clear_table!(tables::AccountsTrie); clear_table!(tables::StoragesTrie); // Reset stage checkpoints so the pipeline rebuilds everything info!(target: "reth::cli", "Resetting stage checkpoints"); - let provider_rw = tool.provider_factory.database_provider_rw()?; + let provider_rw = factory.database_provider_rw()?; for stage in [ StageId::SenderRecovery, StageId::TransactionLookup, @@ -321,15 +360,12 @@ impl Command { provider_rw.save_stage_checkpoint_progress(StageId::MerkleExecute, vec![])?; provider_rw.commit()?; - info!(target: "reth::cli", "Recomputable tables cleared and checkpoints reset"); + info!(target: "reth::cli", "Recomputable tables cleared"); Ok(()) } - /// Creates a compacted copy of the MDBX database to `/../db_compact/`. - /// - /// Returns the path to the compacted copy. The caller must swap it with the - /// original after dropping the database handle. - pub fn compact_mdbx(db: &mdbx::DatabaseEnv) -> eyre::Result { + /// Creates a compacted copy of the MDBX database. + fn compact_mdbx(db: &mdbx::DatabaseEnv) -> eyre::Result<()> { let db_path = db.path(); let compact_path = db_path.with_file_name("db_compact"); @@ -355,6 +391,28 @@ impl Command { } info!(target: "reth::cli", "MDBX compaction complete"); - Ok(compact_path) + Ok(()) + } + + /// Swaps the original MDBX database with a compacted copy. + fn swap_compacted_db( + db_path: &std::path::Path, + compact_path: &std::path::Path, + ) -> eyre::Result<()> { + let backup_path = db_path.with_file_name("db_pre_compact"); + + info!(target: "reth::cli", ?db_path, ?compact_path, "Swapping compacted database"); + + std::fs::rename(db_path, &backup_path)?; + + if let Err(e) = std::fs::rename(compact_path, db_path) { + let _ = std::fs::rename(&backup_path, db_path); + return Err(e.into()); + } + + std::fs::remove_dir_all(&backup_path)?; + + info!(target: "reth::cli", "Database compaction swap complete"); + Ok(()) } } diff --git a/crates/cli/commands/src/db/mod.rs b/crates/cli/commands/src/db/mod.rs index 092549bf9ce..759bcdc9c94 100644 --- a/crates/cli/commands/src/db/mod.rs +++ b/crates/cli/commands/src/db/mod.rs @@ -1,28 +1,14 @@ use crate::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs}; -use alloy_primitives::B256; use clap::{Parser, Subcommand}; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_cli::chainspec::ChainSpecParser; use reth_cli_runner::CliContext; -use reth_consensus::noop::NoopConsensus; -use reth_db::{ - version::{get_db_version, DatabaseVersionError, DB_VERSION}, - DatabaseEnv, -}; +use reth_db::version::{get_db_version, DatabaseVersionError, DB_VERSION}; use reth_db_common::DbTool; -use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader}; -use reth_evm::noop::NoopEvmConfig; -use reth_node_builder::NodeTypesWithDBAdapter; -use reth_provider::StageCheckpointReader; -use reth_stages::{sets::DefaultStages, Pipeline}; -use reth_stages_types::StageId; -use reth_static_file::StaticFileProducer; use std::{ io::{self, Write}, sync::Arc, }; -use tokio::sync::watch; -use tracing::info; mod account_storage; mod checksum; mod clear; @@ -253,56 +239,14 @@ impl> Command let Environment { provider_factory, config, .. } = self.env.init::(AccessRights::RW, ctx.task_executor.clone())?; - let tool = DbTool::new(provider_factory.clone())?; - command.execute(&tool)?; - - // Run pipeline to rebuild senders, indices, and trie - info!(target: "reth::cli", "Running pipeline to rebuild cleared tables"); - - let tip = provider_factory - .provider()? - .get_stage_checkpoint(StageId::Execution)? - .map(|c| c.block_number) - .unwrap_or(0); - - let (_tip_tx, tip_rx) = watch::channel(B256::ZERO); - - let mut pipeline = Pipeline::>::builder() - .with_max_block(tip) - .add_stages(DefaultStages::new( - provider_factory.clone(), - tip_rx, - Arc::new(NoopConsensus::default()), - NoopHeaderDownloader::default(), - NoopBodiesDownloader::default(), - NoopEvmConfig::::default(), - config.stages.clone(), - config.prune.segments.clone(), - None, - )) - .build( - provider_factory.clone(), - StaticFileProducer::new( - provider_factory.clone(), - config.prune.segments.clone(), - ), - ); + // Migrate changesets+receipts, clear tables, compact MDBX + command.execute::(provider_factory).await?; - pipeline.run().await?; - - info!(target: "reth::cli", "Pipeline finished, compacting MDBX"); - - // Compact MDBX while DB handle is still open - migrate_v2::Command::compact_mdbx(tool.provider_factory.db_ref())?; - - // Drop everything to release the DB handle - drop(pipeline); - drop(tool); - drop(provider_factory); + // Reopen DB after compaction swap and run pipeline to rebuild + let Environment { provider_factory, .. } = + self.env.init::(AccessRights::RW, ctx.task_executor.clone())?; - // Swap compacted copy in - let compact_path = db_path.with_file_name("db_compact"); - migrate_v2::Command::swap_compacted_db(&db_path, &compact_path)?; + migrate_v2::Command::run_pipeline::(provider_factory, &config).await?; } } From 8642a2901e74f1a7bdf71456c25168798be84d01 Mon Sep 17 00:00:00 2001 From: klkvr Date: Wed, 15 Apr 2026 16:49:16 +0400 Subject: [PATCH 10/11] rm pipeline --- crates/cli/commands/src/db/migrate_v2.rs | 59 +----------------------- crates/cli/commands/src/db/mod.rs | 8 +--- 2 files changed, 2 insertions(+), 65 deletions(-) diff --git a/crates/cli/commands/src/db/migrate_v2.rs b/crates/cli/commands/src/db/migrate_v2.rs index 9c65cbcaf74..61268f72032 100644 --- a/crates/cli/commands/src/db/migrate_v2.rs +++ b/crates/cli/commands/src/db/migrate_v2.rs @@ -5,10 +5,7 @@ //! state), compacts MDBX, then runs the pipeline to rebuild them. use crate::common::CliNodeTypes; -use alloy_primitives::B256; use clap::Parser; -use reth_config::Config; -use reth_consensus::noop::NoopConsensus; use reth_db::{ mdbx::{self, ffi}, models::StorageBeforeTx, @@ -21,8 +18,6 @@ use reth_db_api::{ tables, transaction::{DbTx, DbTxMut}, }; -use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader}; -use reth_evm::noop::NoopEvmConfig; use reth_node_builder::NodeTypesWithDBAdapter; use reth_provider::{ providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory, MetadataProvider, @@ -30,13 +25,9 @@ use reth_provider::{ StaticFileProviderFactory, StaticFileWriter, StorageSettings, }; use reth_prune_types::PruneSegment; -use reth_stages::{sets::DefaultStages, Pipeline}; use reth_stages_types::{StageCheckpoint, StageId}; -use reth_static_file::StaticFileProducer; use reth_static_file_types::StaticFileSegment; use reth_storage_api::StageCheckpointReader; -use std::sync::Arc; -use tokio::sync::watch; use tracing::info; /// `reth db migrate-v2` command @@ -50,7 +41,6 @@ impl Command { /// 2. Flip `StorageSettings` to v2 /// 3. Clear recomputable MDBX tables + reset stage checkpoints /// 4. Compact MDBX - /// 5. Run pipeline to rebuild senders, indices, and trie pub async fn execute( self, provider_factory: ProviderFactory>, @@ -89,7 +79,6 @@ impl Command { } drop(provider); - info!(target: "reth::cli", "Preflight checks passed"); // === Phase 1: Migrate changesets → static files === Self::migrate_account_changesets(&provider_factory, tip)?; @@ -124,53 +113,7 @@ impl Command { // The caller will reopen the environment and run the pipeline. // We return here — the pipeline step is handled in mod.rs after // reopening the database with the compacted copy. - info!(target: "reth::cli", "Migration data phases complete"); - Ok(()) - } - - /// Builds and runs the pipeline to rebuild cleared tables. - /// - /// Must be called after the database has been compacted and reopened. - pub async fn run_pipeline( - provider_factory: ProviderFactory>, - config: &Config, - ) -> eyre::Result<()> - where - N::Primitives: reth_primitives_traits::NodePrimitives< - Receipt: reth_db_api::table::Value + reth_codecs::Compact, - >, - { - let tip = provider_factory - .provider()? - .get_stage_checkpoint(StageId::Execution)? - .map(|c| c.block_number) - .unwrap_or(0); - - info!(target: "reth::cli", tip, "Running pipeline to rebuild tables"); - - let (_tip_tx, tip_rx) = watch::channel(B256::ZERO); - - let mut pipeline = Pipeline::>::builder() - .with_max_block(tip) - .add_stages(DefaultStages::new( - provider_factory.clone(), - tip_rx, - Arc::new(NoopConsensus::default()), - NoopHeaderDownloader::default(), - NoopBodiesDownloader::default(), - NoopEvmConfig::::default(), - config.stages.clone(), - config.prune.segments.clone(), - None, - )) - .build( - provider_factory.clone(), - StaticFileProducer::new(provider_factory, config.prune.segments.clone()), - ); - - pipeline.run().await?; - - info!(target: "reth::cli", "Pipeline finished"); + info!(target: "reth::cli", "Migration complete. You should now restart the node and let it run the pipeline to rebuild the remaining data."); Ok(()) } diff --git a/crates/cli/commands/src/db/mod.rs b/crates/cli/commands/src/db/mod.rs index 759bcdc9c94..cb3f08f7340 100644 --- a/crates/cli/commands/src/db/mod.rs +++ b/crates/cli/commands/src/db/mod.rs @@ -236,17 +236,11 @@ impl> Command }); } Subcommands::MigrateV2(command) => { - let Environment { provider_factory, config, .. } = + let Environment { provider_factory, .. } = self.env.init::(AccessRights::RW, ctx.task_executor.clone())?; // Migrate changesets+receipts, clear tables, compact MDBX command.execute::(provider_factory).await?; - - // Reopen DB after compaction swap and run pipeline to rebuild - let Environment { provider_factory, .. } = - self.env.init::(AccessRights::RW, ctx.task_executor.clone())?; - - migrate_v2::Command::run_pipeline::(provider_factory, &config).await?; } } From 9f4b71addd0bbc9f496356a0de554ea59d899c77 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Wed, 15 Apr 2026 19:20:36 +0200 Subject: [PATCH 11/11] book --- docs/vocs/docs/pages/cli/SUMMARY.mdx | 1 + docs/vocs/docs/pages/cli/reth/db.mdx | 1 + .../docs/pages/cli/reth/db/migrate-v2.mdx | 166 ++++++++++++++++++ docs/vocs/sidebar-cli-reth.ts | 4 + 4 files changed, 172 insertions(+) create mode 100644 docs/vocs/docs/pages/cli/reth/db/migrate-v2.mdx diff --git a/docs/vocs/docs/pages/cli/SUMMARY.mdx b/docs/vocs/docs/pages/cli/SUMMARY.mdx index 75ff441414e..5244e2db1d1 100644 --- a/docs/vocs/docs/pages/cli/SUMMARY.mdx +++ b/docs/vocs/docs/pages/cli/SUMMARY.mdx @@ -40,6 +40,7 @@ - [`reth db stage-checkpoints set`](./reth/db/stage-checkpoints/set.mdx) - [`reth db account-storage`](./reth/db/account-storage.mdx) - [`reth db state`](./reth/db/state.mdx) + - [`reth db migrate-v2`](./reth/db/migrate-v2.mdx) - [`reth download`](./reth/download.mdx) - [`reth snapshot-manifest`](./reth/snapshot-manifest.mdx) - [`reth stage`](./reth/stage.mdx) diff --git a/docs/vocs/docs/pages/cli/reth/db.mdx b/docs/vocs/docs/pages/cli/reth/db.mdx index 900a97a23de..5996e58597f 100644 --- a/docs/vocs/docs/pages/cli/reth/db.mdx +++ b/docs/vocs/docs/pages/cli/reth/db.mdx @@ -26,6 +26,7 @@ Commands: stage-checkpoints `reth db stage-checkpoints` subcommand account-storage Gets storage size information for an account state Gets account state and storage at a specific block + migrate-v2 Migrate storage layout from v1 (MDBX-only) to v2 (static files + RocksDB) help Print this message or the help of the given subcommand(s) Options: diff --git a/docs/vocs/docs/pages/cli/reth/db/migrate-v2.mdx b/docs/vocs/docs/pages/cli/reth/db/migrate-v2.mdx new file mode 100644 index 00000000000..c5efcb691b3 --- /dev/null +++ b/docs/vocs/docs/pages/cli/reth/db/migrate-v2.mdx @@ -0,0 +1,166 @@ +# reth db migrate-v2 + +Migrate storage layout from v1 (MDBX-only) to v2 (static files + RocksDB) + +```bash +$ reth db migrate-v2 --help +``` +```txt +Usage: reth db migrate-v2 [OPTIONS] + +Options: + -h, --help + Print help (see a summary with '-h') + +Datadir: + --chain + The chain this node is running. + Possible values are either a built-in chain or the path to a chain specification file. + + Built-in chains: + mainnet, sepolia, holesky, hoodi, dev + + [default: mainnet] + +Logging: + --log.stdout.format + The format to use for logs written to stdout + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + [default: terminal] + + --log.stdout.filter + The filter to use for logs written to stdout + + [default: ""] + + --log.file.format + The format to use for logs written to the log file + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + [default: terminal] + + --log.file.filter + The filter to use for logs written to the log file + + [default: debug] + + --log.file.directory + The path to put log files in + + [default: /logs] + + --log.file.name + The prefix name of the log files + + [default: reth.log] + + --log.file.max-size + The maximum size (in MB) of one log file + + [default: 200] + + --log.file.max-files + The maximum amount of log files that will be stored. If set to 0, background file logging is disabled. + + Default: 5 for `node` command, 0 for non-node utility subcommands. + + --log.journald + Write logs to journald + + --log.journald.filter + The filter to use for logs written to journald + + [default: error] + + --color + Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting + + Possible values: + - always: Colors on + - auto: Auto-detect + - never: Colors off + + [default: always] + + --logs-otlp[=] + Enable `Opentelemetry` logs export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/logs` - gRPC: `http://localhost:4317` + + Example: --logs-otlp=http://collector:4318/v1/logs + + [env: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=] + + --logs-otlp.filter + Set a filter directive for the OTLP logs exporter. This controls the verbosity of logs sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --logs-otlp.filter=info,reth=debug + + Defaults to INFO if not specified. + + [default: info] + +Display: + -v, --verbosity... + Set the minimum log level. + + -v Errors + -vv Warnings + -vvv Info + -vvvv Debug + -vvvvv Traces (warning: very verbose!) + + -q, --quiet + Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces and logs. + + - `http`: expects endpoint path to end with `/v1/traces` or `/v1/logs` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] +``` \ No newline at end of file diff --git a/docs/vocs/sidebar-cli-reth.ts b/docs/vocs/sidebar-cli-reth.ts index 0fa4790a553..a1386ea2e35 100644 --- a/docs/vocs/sidebar-cli-reth.ts +++ b/docs/vocs/sidebar-cli-reth.ts @@ -193,6 +193,10 @@ export const rethCliSidebar: SidebarItem = { { text: "reth db state", link: "/cli/reth/db/state" + }, + { + text: "reth db migrate-v2", + link: "/cli/reth/db/migrate-v2" } ] },