From 83f14964a60d05f734ec06f7841f975214bc286b Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Wed, 5 Jul 2023 15:29:37 -0400 Subject: [PATCH] fix: build failing --- Cargo.lock | 4 - components/hord-cli/Cargo.toml | 2 +- components/hord-cli/src/cli/mod.rs | 72 ++------- components/hord-cli/src/config/mod.rs | 1 - components/hord-cli/src/hord/mod.rs | 17 ++ components/hord-cli/src/hord/ordinals.rs | 5 +- components/hord-cli/src/scan/bitcoin.rs | 18 +-- components/hord-cli/src/service/mod.rs | 166 +++++++++++++++----- components/hord-cli/src/service/runloops.rs | 4 +- components/hord-cli/src/storage/mod.rs | 3 +- 10 files changed, 169 insertions(+), 123 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd2ba88d..6c158039 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -445,13 +445,11 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" name = "chainhook-sdk" version = "0.5.0" dependencies = [ - "anyhow", "base58 0.2.0", "base64", "bitcoincore-rpc", "bitcoincore-rpc-json", "chainhook-types 1.0.7", - "chrono", "clarinet-files", "clarinet-utils", "crossbeam-channel 0.5.8", @@ -464,8 +462,6 @@ dependencies = [ "rand 0.8.5", "reqwest", "rocket", - "rocksdb", - "rusqlite", "schemars", "serde", "serde-hex", diff --git a/components/hord-cli/Cargo.toml b/components/hord-cli/Cargo.toml index e7261b44..763cca1e 100644 --- a/components/hord-cli/Cargo.toml +++ b/components/hord-cli/Cargo.toml @@ -14,7 +14,7 @@ redis = "0.21.5" serde-redis = "0.12.0" hex = "0.4.3" rand = "0.8.5" -chainhook-sdk = { version = "0.5.0", default-features = false, features = ["ordinals", "zeromq"], path = "../../../chainhook/components/chainhook-sdk" } +chainhook-sdk = { version = "0.5.0", default-features = false, features = ["zeromq"], path = "../../../chainhook/components/chainhook-sdk" } chainhook-types = { version = "1.0.6", path = "../../../chainhook/components/chainhook-types-rs" } clarinet-files = "1.0.1" hiro-system-kit = "0.1.0" diff --git a/components/hord-cli/src/cli/mod.rs b/components/hord-cli/src/cli/mod.rs index 59bf61bb..cc5c2be9 100644 --- a/components/hord-cli/src/cli/mod.rs +++ b/components/hord-cli/src/cli/mod.rs @@ -1,35 +1,31 @@ use crate::config::generator::generate_config; use crate::config::{Config, PredicatesApi}; -use crate::scan::bitcoin::scan_bitcoin_chainstate_via_rpc_using_predicate; use crate::service::Service; -use crate::storage::get_last_block_height_inserted; -use chainhook_sdk::chainhooks::types::ChainhookFullSpecification; -use chainhook_sdk::hord::db::{ +use crate::db::{ delete_data_in_hord_db, find_last_block_inserted, find_lazy_block_at_block_height, find_watched_satpoint_for_inscription, initialize_hord_db, open_readonly_hord_db_conn, open_readonly_hord_db_conn_rocks_db, open_readwrite_hord_db_conn, open_readwrite_hord_db_conn_rocks_db, retrieve_satoshi_point_using_lazy_storage, }; -use chainhook_sdk::hord::{ +use crate::hord::{ new_traversals_lazy_cache, retrieve_inscribed_satoshi_points_from_block, update_storage_and_augment_bitcoin_block_with_inscription_transfer_data, Storage, }; +use chainhook_sdk::chainhooks::types::ChainhookFullSpecification; use chainhook_sdk::indexer; use chainhook_sdk::indexer::bitcoin::{ download_and_parse_block_with_retry, retrieve_block_hash_with_retry, }; use chainhook_sdk::observer::BitcoinConfig; use chainhook_sdk::utils::Context; -use chainhook_types::{BitcoinBlockData, BitcoinNetwork, BlockIdentifier, TransactionIdentifier}; +use chainhook_types::{BitcoinBlockData, BlockIdentifier, TransactionIdentifier}; use clap::{Parser, Subcommand}; -use ctrlc; use hiro_system_kit; use std::collections::BTreeMap; use std::io::{BufReader, Read}; use std::path::PathBuf; use std::process; -use std::sync::mpsc::Sender; use std::sync::Arc; #[derive(Parser, Debug)] @@ -153,26 +149,6 @@ enum HordDbCommand { Check(CheckDbCommand), } -#[derive(Subcommand, PartialEq, Clone, Debug)] -enum StacksCommand { - /// Db maintenance related commands - #[clap(subcommand)] - Db(StacksDbCommand), -} - -#[derive(Subcommand, PartialEq, Clone, Debug)] -enum StacksDbCommand { - /// Check integrity - #[clap(name = "check", bin_name = "check")] - Check(CheckDbCommand), - /// Update database using latest Stacks archive file - #[clap(name = "update", bin_name = "update")] - Update(UpdateDbCommand), - /// Retrieve a block from the Stacks db - #[clap(name = "get", bin_name = "get")] - GetBlock(GetBlockDbCommand), -} - #[derive(Subcommand, PartialEq, Clone, Debug)] enum ScanCommand { /// Compute ordinal number of the 1st satoshi of the 1st input of a given transaction @@ -307,30 +283,6 @@ struct CheckDbCommand { pub config_path: Option, } -#[derive(Parser, PartialEq, Clone, Debug)] -struct UpdateDbCommand { - /// Load config file path - #[clap(long = "config-path")] - pub config_path: Option, -} - -#[derive(Parser, PartialEq, Clone, Debug)] -struct GetBlockDbCommand { - /// Block index to retrieve - #[clap(long = "block-height")] - pub block_height: u64, - /// Load config file path - #[clap(long = "config-path")] - pub config_path: Option, -} - -#[derive(Parser, PartialEq, Clone, Debug)] -struct InitHordDbCommand { - /// Load config file path - #[clap(long = "config-path")] - pub config_path: Option, -} - pub fn main() { let logger = hiro_system_kit::log::setup_logger(); let _guard = hiro_system_kit::log::setup_global_logger(logger.clone()); @@ -444,6 +396,7 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> { } None => { let event_observer_config = config.get_event_observer_config(); + let hord_config = config.get_hord_config(); let bitcoin_config = event_observer_config.get_bitcoin_config(); let block = fetch_and_standardize_block(cmd.block_height, &bitcoin_config, &ctx) @@ -453,7 +406,7 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> { let _traversals = retrieve_inscribed_satoshi_points_from_block( &block, None, - event_observer_config.hord_config.as_ref().unwrap(), + &hord_config, &traversals_cache, &ctx, ); @@ -621,13 +574,12 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> { "Cleaning hord_db: {} blocks dropped", cmd.end_block - cmd.start_block + 1 ); - } - // HordDbCommand::Patch(_cmd) => { - // unimplemented!() - // } - // HordDbCommand::Migrate(_cmd) => { - // unimplemented!() - // } + } // HordDbCommand::Patch(_cmd) => { + // unimplemented!() + // } + // HordDbCommand::Migrate(_cmd) => { + // unimplemented!() + // } } Ok(()) } diff --git a/components/hord-cli/src/config/mod.rs b/components/hord-cli/src/config/mod.rs index 57a63b72..e168822b 100644 --- a/components/hord-cli/src/config/mod.rs +++ b/components/hord-cli/src/config/mod.rs @@ -136,7 +136,6 @@ impl Config { cache_path: self.storage.working_dir.clone(), bitcoin_network: self.network.bitcoin_network.clone(), stacks_network: self.network.stacks_network.clone(), - hord_config: None, } } diff --git a/components/hord-cli/src/hord/mod.rs b/components/hord-cli/src/hord/mod.rs index 2195849b..a16d6199 100644 --- a/components/hord-cli/src/hord/mod.rs +++ b/components/hord-cli/src/hord/mod.rs @@ -977,3 +977,20 @@ fn test_identify_next_output_index_destination() { SatPosition::Fee(0) ); } + +#[test] +fn test_ordinal_inscription_parsing() { + let bytes = hex::decode("208737bc46923c3e64c7e6768c0346879468bf3aba795a5f5f56efca288f50ed2aac0063036f7264010118746578742f706c61696e3b636861727365743d7574662d38004c9948656c6c6f2030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030300a68").unwrap(); + + let script = Script::from(bytes); + let parser = InscriptionParser { + instructions: script.instructions().peekable(), + }; + + let inscription = match parser.parse_script() { + Ok(inscription) => inscription, + Err(_) => panic!(), + }; + + println!("{:?}", inscription); +} diff --git a/components/hord-cli/src/hord/ordinals.rs b/components/hord-cli/src/hord/ordinals.rs index e391c3cc..163c2eb2 100644 --- a/components/hord-cli/src/hord/ordinals.rs +++ b/components/hord-cli/src/hord/ordinals.rs @@ -1,7 +1,4 @@ -use std::{ - collections::HashMap, - sync::mpsc::{channel, Sender}, -}; +use std::sync::mpsc::{channel, Sender}; use super::HordConfig; diff --git a/components/hord-cli/src/scan/bitcoin.rs b/components/hord-cli/src/scan/bitcoin.rs index cfd8a10d..6dbb09d8 100644 --- a/components/hord-cli/src/scan/bitcoin.rs +++ b/components/hord-cli/src/scan/bitcoin.rs @@ -1,5 +1,13 @@ use crate::archive::download_ordinals_dataset_if_required; use crate::config::{Config, PredicatesApi}; +use crate::db::{ + find_all_inscriptions_in_block, get_any_entry_in_ordinal_activities, open_readonly_hord_db_conn, +}; +use crate::hord::{ + get_inscriptions_revealed_in_block, + update_storage_and_augment_bitcoin_block_with_inscription_reveal_data, + update_storage_and_augment_bitcoin_block_with_inscription_transfer_data, Storage, +}; use crate::service::{ open_readwrite_predicates_db_conn_or_panic, update_predicate_status, PredicateStatus, ScanningData, @@ -11,14 +19,6 @@ use chainhook_sdk::chainhooks::bitcoin::{ BitcoinChainhookOccurrence, BitcoinTriggerChainhook, }; use chainhook_sdk::chainhooks::types::{BitcoinChainhookSpecification, BitcoinPredicateType}; -use chainhook_sdk::hord::db::{ - find_all_inscriptions_in_block, get_any_entry_in_ordinal_activities, open_readonly_hord_db_conn, -}; -use chainhook_sdk::hord::{ - get_inscriptions_revealed_in_block, - update_storage_and_augment_bitcoin_block_with_inscription_reveal_data, - update_storage_and_augment_bitcoin_block_with_inscription_transfer_data, Storage, -}; use chainhook_sdk::indexer; use chainhook_sdk::indexer::bitcoin::{ download_and_parse_block_with_retry, retrieve_block_hash_with_retry, @@ -88,7 +88,7 @@ pub async fn scan_bitcoin_chainstate_via_rpc_using_predicate( let mut blocks_scanned = 0; let mut actions_triggered = 0; - let mut occurrences_found = 0u64; + let occurrences_found = 0u64; let mut err_count = 0; let event_observer_config = config.get_event_observer_config(); diff --git a/components/hord-cli/src/service/mod.rs b/components/hord-cli/src/service/mod.rs index e2aefb64..69f591a9 100644 --- a/components/hord-cli/src/service/mod.rs +++ b/components/hord-cli/src/service/mod.rs @@ -2,7 +2,11 @@ mod http_api; mod runloops; use crate::config::{Config, PredicatesApi, PredicatesApiConfig}; -use crate::hord::should_sync_hord_db; +use crate::db::open_readwrite_hord_dbs; +use crate::hord::{ + new_traversals_lazy_cache, revert_hord_db_with_augmented_bitcoin_block, should_sync_hord_db, + update_hord_db_and_augment_bitcoin_block, +}; use crate::scan::bitcoin::process_block_with_predicates; use crate::service::http_api::{load_predicates_from_redis, start_predicate_api_server}; use crate::service::runloops::start_bitcoin_scan_runloop; @@ -12,12 +16,15 @@ use chainhook_sdk::chainhooks::types::{ }; use chainhook_sdk::chainhooks::types::ChainhookSpecification; +use chainhook_sdk::chainhooks::types::{ChainhookConfig, ChainhookFullSpecification}; use chainhook_sdk::observer::{start_event_observer, ObserverEvent}; use chainhook_sdk::utils::Context; -use chainhook_types::BitcoinBlockSignaling; +use chainhook_types::{BitcoinBlockSignaling, BitcoinChainEvent}; +use hiro_system_kit::slog; use redis::{Commands, Connection}; use std::sync::mpsc::channel; +use std::sync::Arc; pub struct Service { config: Config, @@ -102,32 +109,25 @@ impl Service { let mut event_observer_config = self.config.get_event_observer_config(); event_observer_config.chainhook_config = Some(chainhook_config); - // Download and ingest a Ordinal dump, if hord is enabled - if !hord_disabled { - // TODO: add flag - // let _ = download_ordinals_dataset_if_required(&mut self.config, &self.ctx).await; - info!( - self.ctx.expect_logger(), - "Ordinal indexing is enabled by default, checking index... (use --no-hord to disable ordinals)" - ); + let hord_config = self.config.get_hord_config(); - if let Some((start_block, end_block)) = should_sync_hord_db(&self.config, &self.ctx)? { - if start_block == 0 { - info!( - self.ctx.expect_logger(), - "Initializing hord indexing from block #{}", start_block - ); - } else { - info!( - self.ctx.expect_logger(), - "Resuming hord indexing from block #{}", start_block - ); - } + if let Some((start_block, end_block)) = should_sync_hord_db(&self.config, &self.ctx)? { + if start_block == 0 { + info!( + self.ctx.expect_logger(), + "Initializing hord indexing from block #{}", start_block + ); + } else { + info!( + self.ctx.expect_logger(), + "Resuming hord indexing from block #{}", start_block + ); + } - let (tx, rx) = channel(); + let (tx, rx) = channel(); - let mut moved_event_observer_config = event_observer_config.clone(); - let moved_ctx = self.ctx.clone(); + let mut moved_event_observer_config = event_observer_config.clone(); + let moved_ctx = self.ctx.clone(); let _ = hiro_system_kit::thread_named("Initial predicate processing") .spawn(move || { @@ -154,21 +154,23 @@ impl Service { } } } - }) - .expect("unable to spawn thread"); - - crate::hord::perform_hord_db_update( - start_block, - end_block, - &self.config.get_hord_config(), - &self.config, - Some(tx), - &self.ctx, - ) - .await?; - } + } + }) + .expect("unable to spawn thread"); + + crate::hord::perform_hord_db_update( + start_block, + end_block, + &self.config.get_hord_config(), + &self.config, + Some(tx), + &self.ctx, + ) + .await?; } + let traversals_cache = Arc::new(new_traversals_lazy_cache(hord_config.cache_size)); + // Bitcoin scan operation threadpool let (bitcoin_scan_op_tx, bitcoin_scan_op_rx) = crossbeam_channel::unbounded(); let ctx = self.ctx.clone(); @@ -346,8 +348,92 @@ impl Service { } } } - ObserverEvent::BitcoinChainEvent((_chain_update, _report)) => { - debug!(self.ctx.expect_logger(), "Bitcoin update not stored"); + ObserverEvent::BitcoinChainEvent((mut chain_update, _report)) => { + // + let (blocks_db, inscriptions_db_conn_rw) = match open_readwrite_hord_dbs( + &self.config.expected_cache_path(), + &self.ctx, + ) { + Ok(dbs) => dbs, + Err(e) => { + self.ctx.try_log(|logger| { + slog::error!(logger, "Unable to open readwtite connection: {e}",) + }); + continue; + } + }; + + // Update Chain event before propagation + match chain_update { + BitcoinChainEvent::ChainUpdatedWithBlocks(ref mut data) => { + for block in data.new_blocks.iter_mut() { + if let Err(e) = update_hord_db_and_augment_bitcoin_block( + block, + &blocks_db, + &inscriptions_db_conn_rw, + true, + &hord_config, + &traversals_cache, + &self.ctx, + ) { + self.ctx.try_log(|logger| { + slog::error!( + logger, + "Unable to insert bitcoin block {} in hord_db: {e}", + block.block_identifier.index + ) + }); + } + } + } + BitcoinChainEvent::ChainUpdatedWithReorg(ref mut data) => { + for block in data.blocks_to_rollback.iter() { + if let Err(e) = revert_hord_db_with_augmented_bitcoin_block( + block, + &blocks_db, + &inscriptions_db_conn_rw, + &self.ctx, + ) { + self.ctx.try_log(|logger| { + slog::error!( + logger, + "Unable to rollback bitcoin block {}: {e}", + block.block_identifier + ) + }); + } + } + + for block in data.blocks_to_apply.iter_mut() { + if let Err(e) = update_hord_db_and_augment_bitcoin_block( + block, + &blocks_db, + &inscriptions_db_conn_rw, + true, + &hord_config, + &traversals_cache, + &self.ctx, + ) { + self.ctx.try_log(|logger| { + slog::error!( + logger, + "Unable to apply bitcoin block {} with hord_db: {e}", + block.block_identifier.index + ) + }); + } + } + } + }; + self.ctx.try_log(|logger| { + slog::info!( + logger, + "Flushing traversals_cache ({} entries)", + traversals_cache.len() + ) + }); + + // traversals_cache.clear(); } ObserverEvent::Terminate => { info!(self.ctx.expect_logger(), "Terminating runloop"); diff --git a/components/hord-cli/src/service/runloops.rs b/components/hord-cli/src/service/runloops.rs index 9218c3d8..0c73d8d0 100644 --- a/components/hord-cli/src/service/runloops.rs +++ b/components/hord-cli/src/service/runloops.rs @@ -1,9 +1,7 @@ use std::sync::mpsc::Sender; use chainhook_sdk::{ - chainhooks::types::{ - BitcoinChainhookSpecification, ChainhookSpecification, - }, + chainhooks::types::{BitcoinChainhookSpecification, ChainhookSpecification}, observer::ObserverCommand, utils::Context, }; diff --git a/components/hord-cli/src/storage/mod.rs b/components/hord-cli/src/storage/mod.rs index 557a1563..6389120b 100644 --- a/components/hord-cli/src/storage/mod.rs +++ b/components/hord-cli/src/storage/mod.rs @@ -1,7 +1,8 @@ use std::path::PathBuf; -use chainhook_sdk::{rocksdb::Options, rocksdb::DB, utils::Context}; +use chainhook_sdk::utils::Context; use chainhook_types::{BlockIdentifier, StacksBlockData, StacksBlockUpdate}; +use rocksdb::{Options, DB}; fn get_db_default_options() -> Options { let mut opts = Options::default();