feat(drive-abci): debugging tool to replay abci requests

packages/rs-drive-abci/.env.testnet

@@ -33,21 +33,21 @@ CORE_CHECK_TX_JSON_RPC_PASSWORD=password
 INITIAL_CORE_CHAINLOCKED_HEIGHT=1243
 
 # https://github.com/dashevo/dashcore-lib/blob/286c33a9d29d33f05d874c47a9b33764a0be0cf1/lib/constants/index.js#L42-L57
-VALIDATOR_SET_QUORUM_TYPE=llmq_25_67
-VALIDATOR_SET_QUORUM_SIZE=25
+VALIDATOR_SET_QUORUM_TYPE=6
+VALIDATOR_SET_QUORUM_SIZE=100
 VALIDATOR_SET_QUORUM_WINDOW=24
 VALIDATOR_SET_QUORUM_ACTIVE_SIGNERS=24
 VALIDATOR_SET_QUORUM_ROTATION=false
 VALIDATOR_SET_ROTATION_BLOCK_COUNT=64
 
-CHAIN_LOCK_QUORUM_TYPE=llmq_50_60
-CHAIN_LOCK_QUORUM_SIZE=50
+CHAIN_LOCK_QUORUM_TYPE=1
+CHAIN_LOCK_QUORUM_SIZE=400
 CHAIN_LOCK_QUORUM_WINDOW=24
 CHAIN_LOCK_QUORUM_ACTIVE_SIGNERS=24
 CHAIN_LOCK_QUORUM_ROTATION=false
 
-INSTANT_LOCK_QUORUM_TYPE=llmq_60_75
-INSTANT_LOCK_QUORUM_SIZE=50
+INSTANT_LOCK_QUORUM_TYPE=5
+INSTANT_LOCK_QUORUM_SIZE=60
 INSTANT_LOCK_QUORUM_WINDOW=288
 INSTANT_LOCK_QUORUM_ACTIVE_SIGNERS=32
 INSTANT_LOCK_QUORUM_ROTATION=true
@@ -77,9 +77,9 @@ MASTERNODE_REWARD_SHARES_SECOND_PUBLIC_KEY=02bf55f97f189895da29824781053140ee66b
 WITHDRAWALS_MASTER_PUBLIC_KEY=027057cdf58628635ef7b75e6b6c90dd996a16929cd68130e16b9328d429e5e03a
 WITHDRAWALS_SECOND_PUBLIC_KEY=022084d827fea4823a69aa7c8d3e02fe780eaa0ef1e5e9841af395ba7e40465ab6
 
-EPOCH_TIME_LENGTH_S=788400
+EPOCH_TIME_LENGTH_S=3600
 
-CHAIN_ID=devnet
+CHAIN_ID=dash-testnet-51
 BLOCK_SPACING_MS=5000
 
 TOKIO_CONSOLE_ENABLED=false

packages/rs-drive-abci/Cargo.toml

@@ -15,6 +15,7 @@ license = "MIT"
 [dependencies]
 arc-swap = "1.7.0"
 bincode = { version = "=2.0.0-rc.3", features = ["serde"] }
+base64 = { version = "0.22.1", optional = true }
 ciborium = { version = "0.2.2" }
 chrono = "0.4.35"
 serde = { version = "1.0.219", features = ["derive"] }
@@ -54,6 +55,11 @@ tracing-subscriber = { version = "0.3.16", default-features = false, features =
 tenderdash-abci = { git = "https://github.com/dashpay/rs-tenderdash-abci", tag = "v1.5.0-dev.2", features = [
   "grpc",
 ] }
+time = { version = "0.3", optional = true, features = [
+  "macros",
+  "formatting",
+  "serde-human-readable",
+] }
 
 lazy_static = "1.4.0"
 itertools = { version = "0.13" }
@@ -75,9 +81,9 @@ tokio = { version = "1.40", features = [
 tokio-util = { version = "0.7" }
 derive_more = { version = "1.0", features = ["from", "deref", "deref_mut"] }
 async-trait = "0.1.77"
+ron = { version = "0.12", optional = true }
 console-subscriber = { version = "0.4", optional = true }
 bls-signatures = { git = "https://github.com/dashpay/bls-signatures", rev = "0842b17583888e8f46c252a4ee84cdfd58e0546f", optional = true }
-
 [dev-dependencies]
 bs58 = { version = "0.5.0" }
 base64 = "0.22.1"
@@ -104,7 +110,7 @@ bls-signatures = { git = "https://github.com/dashpay/bls-signatures", rev = "084
 mockall = { version = "0.13" }
 
 # For tests of grovedb verify
-rocksdb = { version = "0.23.0" }
+rocksdb = { version = "0.24.0" }
 integer-encoding = { version = "4.0.0" }
 
 [features]
@@ -113,7 +119,8 @@ mocks = ["mockall", "drive/fixtures-and-mocks", "bls-signatures"]
 console = ["console-subscriber", "tokio/tracing"]
 testing-config = []
 grovedbg = ["drive/grovedbg"]
-
+# `abci-server replay` command
+replay = ["dep:ron", "dep:time", "tenderdash-abci/serde"]
 [[bin]]
 name = "drive-abci"
 path = "src/main.rs"

packages/rs-drive-abci/src/abci/handler/process_proposal.rs

@@ -33,12 +33,11 @@ where
     if let Some(block_execution_context) = block_execution_context_guard.as_mut() {
         // We are already in a block, or in init chain.
         // This only makes sense if we were the proposer unless we are at a future round
-        if block_execution_context.block_state_info().round() != (request.round as u32) {
+        let block_state_info = block_execution_context.block_state_info();
+        if block_state_info.round() != (request.round as u32) {
             // We were not the proposer, and we should process something new
             drop_block_execution_context = true;
-        } else if let Some(current_block_hash) =
-            block_execution_context.block_state_info().block_hash()
-        {
+        } else if let Some(current_block_hash) = block_state_info.block_hash() {
             // There is also the possibility that this block already came in, but tenderdash crashed
             // Now tenderdash is sending it again
             if let Some(proposal_info) = block_execution_context.proposer_results() {
@@ -69,6 +68,15 @@ where
             } else {
                 // We are getting a different block hash for a block of the same round
                 // This is a terrible issue
+                tracing::error!(
+                    method = "process_proposal",
+                    block_state_info = ?block_state_info,
+                    "received a process proposal request twice with different hash for height {}/round {}: existing hash {:?}, new hash {:?}",
+                    request.height,
+                    request.round,
+                    current_block_hash,
+                    request.hash,
+                );
                 Err(Error::Abci(AbciError::BadRequest(
                     "received a process proposal request twice with different hash".to_string(),
                 )))?;

packages/rs-drive-abci/src/execution/engine/run_block_proposal/v0/mod.rs

@@ -399,6 +399,7 @@ where
             tracing::trace!(
                 method = "run_block_proposal_v0",
                 app_hash = hex::encode(root_hash),
+                block_hash = hex::encode(block_proposal.block_hash.unwrap_or_default()),
                 platform_state_fingerprint = hex::encode(
                     block_execution_context
                         .block_platform_state()

packages/rs-drive-abci/src/execution/storage/fetch_platform_state/mod.rs

@@ -1,3 +1,5 @@
+//! Fetches execution state from grovedb storage
+
 use crate::error::execution::ExecutionError;
 use crate::error::Error;
 use crate::platform_types::platform::Platform;

packages/rs-drive-abci/src/execution/storage/mod.rs

@@ -1,2 +1,2 @@
-mod fetch_platform_state;
+pub mod fetch_platform_state;
 mod store_platform_state;

packages/rs-drive-abci/src/lib.rs

@@ -50,5 +50,10 @@ pub mod query;
 /// Various utils
 pub mod utils;
 
+/// Replay captured ABCI requests against drive-abci
+#[cfg(feature = "replay")]
+pub mod replay;
 /// Drive server
 pub mod server;
+/// Verification helpers
+pub mod verify;

packages/rs-drive-abci/src/main.rs

@@ -1,13 +1,15 @@
 //! Main server process for RS-Drive-ABCI
 //!
 //! RS-Drive-ABCI server starts a single-threaded server and listens to connections from Tenderdash.
+#[cfg(feature = "replay")]
+use drive_abci::replay::{self, ReplayArgs};
+use drive_abci::verify::verify_grovedb;
 
 use clap::{Parser, Subcommand};
 use dapi_grpc::platform::v0::get_status_request;
 use dapi_grpc::platform::v0::get_status_request::GetStatusRequestV0;
 use dapi_grpc::platform::v0::platform_client::PlatformClient;
 use dapi_grpc::tonic::transport::Uri;
-use dpp::version::PlatformVersion;
 use drive_abci::config::{FromEnv, PlatformConfig};
 use drive_abci::core::wait_for_core_to_sync::v0::wait_for_core_to_sync_v0;
 use drive_abci::logging::{LogBuilder, LogConfig, LogDestination, Loggers};
@@ -16,7 +18,6 @@ use drive_abci::platform_types::platform::Platform;
 use drive_abci::rpc::core::DefaultCoreRPC;
 use drive_abci::{logging, server};
 use itertools::Itertools;
-use std::fs::remove_file;
 #[cfg(all(tokio_unstable, feature = "console"))]
 use std::net::SocketAddr;
 use std::path::PathBuf;
@@ -63,6 +64,11 @@ enum Commands {
     /// Print current software version
     #[command()]
     Version,
+
+    /// Replay ABCI requests captured from drive-abci logs.
+    #[cfg(feature = "replay")]
+    #[command()]
+    Replay(ReplayArgs),
 }
 
 /// Server that accepts connections from Tenderdash, and
@@ -151,8 +157,13 @@ impl Cli {
             }
             Commands::Config => dump_config(&config)?,
             Commands::Status => runtime.block_on(check_status(&config))?,
-            Commands::Verify => verify_grovedb(&config.db_path, true)?,
+            Commands::Verify => drive_abci::verify::run(&config, true)?,
             Commands::Version => print_version(),
+            #[cfg(feature = "replay")]
+            Commands::Replay(args) => {
+                replay::run(config, args, cancel.clone()).map_err(|e| e.to_string())?;
+                return Ok(());
+            }
         };
 
         Ok(())
@@ -331,62 +342,6 @@ async fn check_status(config: &PlatformConfig) -> Result<(), String> {
         .map_err(|e| format!("can't request status: {e}"))
 }
 
-/// Verify GroveDB integrity.
-///
-/// This function will execute GroveDB integrity checks if one of the following conditions is met:
-/// - `force` is `true`
-/// - file `.fsck` in `config.db_path` exists
-///
-/// After successful verification, .fsck file is removed.
-fn verify_grovedb(db_path: &PathBuf, force: bool) -> Result<(), String> {
-    let fsck = PathBuf::from(db_path).join(".fsck");
-
-    if !force {
-        if !fsck.exists() {
-            return Ok(());
-        }
-        tracing::info!(
-            "found {} file, starting grovedb verification",
-            fsck.display()
-        );
-    }
-
-    let grovedb = drive::grovedb::GroveDb::open(db_path).expect("open grovedb");
-    //todo: get platform version instead of taking latest
-    let result = grovedb
-        .visualize_verify_grovedb(
-            None,
-            true,
-            true,
-            &PlatformVersion::latest().drive.grove_version,
-        )
-        .map_err(|e| e.to_string());
-
-    match result {
-        Ok(data) => {
-            for result in data {
-                tracing::warn!(?result, "grovedb verification")
-            }
-            tracing::info!("grovedb verification finished");
-
-            if fsck.exists() {
-                if let Err(e) = remove_file(&fsck) {
-                    tracing::warn!(
-                        error = ?e,
-                        path  =fsck.display().to_string(),
-                        "grovedb verification: cannot remove .fsck file: please remove it manually to avoid running verification again",
-                    );
-                }
-            }
-            Ok(())
-        }
-        Err(e) => {
-            tracing::error!("grovedb verification failed: {}", e);
-            Err(e)
-        }
-    }
-}
-
 /// Print current software version.
 fn print_version() {
     println!("{}", env!("CARGO_PKG_VERSION"));