diff --git a/Cargo.lock b/Cargo.lock index 5d45fae07a7..2a53a0de93e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3588,7 +3588,7 @@ dependencies = [ [[package]] name = "mithril-aggregator" -version = "0.5.119" +version = "0.5.120" dependencies = [ "anyhow", "async-trait", @@ -3745,7 +3745,7 @@ dependencies = [ [[package]] name = "mithril-common" -version = "0.4.92" +version = "0.4.93" dependencies = [ "anyhow", "async-trait", diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index 30bc1fdb7a7..7758bc8539d 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-aggregator" -version = "0.5.119" +version = "0.5.120" description = "A Mithril Aggregator server" authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-aggregator/src/artifact_builder/cardano_database.rs b/mithril-aggregator/src/artifact_builder/cardano_database.rs new file mode 100644 index 00000000000..1e6f54b24a9 --- /dev/null +++ b/mithril-aggregator/src/artifact_builder/cardano_database.rs @@ -0,0 +1,187 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Context}; +use async_trait::async_trait; +use semver::Version; + +use mithril_common::{ + entities::{ + ArtifactsLocations, CardanoDatabaseSnapshot, CardanoDbBeacon, Certificate, + CompressionAlgorithm, ProtocolMessagePartKey, SignedEntityType, + }, + StdResult, +}; + +use crate::artifact_builder::ArtifactBuilder; + +pub struct CardanoDatabaseArtifactBuilder { + db_directory: PathBuf, // TODO: temporary, will be accessed through another dependency instead of direct path. + cardano_node_version: Version, + compression_algorithm: CompressionAlgorithm, +} + +impl CardanoDatabaseArtifactBuilder { + pub fn new( + db_directory: PathBuf, + cardano_node_version: &Version, + compression_algorithm: CompressionAlgorithm, + ) -> Self { + Self { + db_directory, + cardano_node_version: cardano_node_version.clone(), + compression_algorithm, + } + } +} + +#[async_trait] +impl ArtifactBuilder for CardanoDatabaseArtifactBuilder { + async fn compute_artifact( + &self, + beacon: CardanoDbBeacon, + certificate: &Certificate, + ) -> StdResult { + let merkle_root = certificate + .protocol_message + .get_message_part(&ProtocolMessagePartKey::CardanoDatabaseMerkleRoot) + .ok_or(anyhow!( + "Can not find CardanoDatabaseMerkleRoot protocol message part in certificate" + )) + .with_context(|| { + format!( + "Can not compute CardanoDatabase artifact for signed_entity: {:?}", + SignedEntityType::CardanoDatabase(beacon.clone()) + ) + })?; + let total_db_size_uncompressed = compute_uncompressed_database_size(&self.db_directory)?; + + let cardano_database = CardanoDatabaseSnapshot::new( + merkle_root.to_string(), + beacon, + total_db_size_uncompressed, + ArtifactsLocations::default(), // TODO: temporary default locations, will be injected in next PR. + self.compression_algorithm, + &self.cardano_node_version, + ); + + Ok(cardano_database) + } +} + +fn compute_uncompressed_database_size(path: &Path) -> StdResult { + if path.is_file() { + let metadata = std::fs::metadata(path) + .with_context(|| format!("Failed to read metadata for file: {:?}", path))?; + + return Ok(metadata.len()); + } + + if path.is_dir() { + let entries = std::fs::read_dir(path) + .with_context(|| format!("Failed to read directory: {:?}", path))?; + let mut directory_size = 0; + for entry in entries { + let path = entry + .with_context(|| format!("Failed to read directory entry in {:?}", path))? + .path(); + directory_size += compute_uncompressed_database_size(&path)?; + } + + return Ok(directory_size); + } + + Ok(0) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use mithril_common::{ + digesters::DummyImmutablesDbBuilder, + entities::{ProtocolMessage, ProtocolMessagePartKey}, + test_utils::{fake_data, TempDir}, + }; + + use super::*; + + fn get_test_directory(dir_name: &str) -> PathBuf { + TempDir::create("cardano_database", dir_name) + } + + #[test] + fn should_compute_the_size_of_the_uncompressed_database_only_immutable_ledger_and_volatile() { + let test_dir = get_test_directory("should_compute_the_size_of_the_uncompressed_database_only_immutable_ledger_and_volatile"); + + let immutable_trio_file_size = 777; + let ledger_file_size = 6666; + let volatile_file_size = 99; + DummyImmutablesDbBuilder::new(test_dir.as_os_str().to_str().unwrap()) + .with_immutables(&[1, 2]) + .set_immutable_trio_file_size(immutable_trio_file_size) + .with_ledger_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"]) + .set_ledger_file_size(ledger_file_size) + .with_volatile_files(&["437", "537", "637", "737"]) + .set_volatile_file_size(volatile_file_size) + .build(); + let expected_total_size = + (2 * immutable_trio_file_size) + (3 * ledger_file_size) + (4 * volatile_file_size); + + let total_size = compute_uncompressed_database_size(&test_dir).unwrap(); + + assert_eq!(expected_total_size, total_size); + } + + #[tokio::test] + async fn should_compute_valid_artifact() { + let test_dir = get_test_directory("should_compute_valid_artifact"); + + let immutable_trio_file_size = 777; + let ledger_file_size = 6666; + let volatile_file_size = 99; + DummyImmutablesDbBuilder::new(test_dir.as_os_str().to_str().unwrap()) + .with_immutables(&[1]) + .set_immutable_trio_file_size(immutable_trio_file_size) + .with_ledger_files(&["blocks-0.dat"]) + .set_ledger_file_size(ledger_file_size) + .with_volatile_files(&["437"]) + .set_volatile_file_size(volatile_file_size) + .build(); + let expected_total_size = immutable_trio_file_size + ledger_file_size + volatile_file_size; + + let cardano_database_artifact_builder = CardanoDatabaseArtifactBuilder::new( + test_dir, + &Version::parse("1.0.0").unwrap(), + CompressionAlgorithm::Zstandard, + ); + + let beacon = fake_data::beacon(); + let certificate_with_merkle_root = { + let mut protocol_message = ProtocolMessage::new(); + protocol_message.set_message_part( + ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, + "merkleroot".to_string(), + ); + Certificate { + protocol_message, + ..fake_data::certificate("certificate-123".to_string()) + } + }; + + let artifact = cardano_database_artifact_builder + .compute_artifact(beacon.clone(), &certificate_with_merkle_root) + .await + .unwrap(); + + let artifact_expected = CardanoDatabaseSnapshot::new( + "merkleroot".to_string(), + beacon, + expected_total_size, + ArtifactsLocations::default(), + CompressionAlgorithm::Zstandard, + &Version::parse("1.0.0").unwrap(), + ); + + assert_eq!(artifact_expected, artifact); + } +} diff --git a/mithril-aggregator/src/artifact_builder/mod.rs b/mithril-aggregator/src/artifact_builder/mod.rs index ea7ce80a6b7..45c7facc815 100644 --- a/mithril-aggregator/src/artifact_builder/mod.rs +++ b/mithril-aggregator/src/artifact_builder/mod.rs @@ -1,10 +1,12 @@ //! The module used for building artifact +mod cardano_database; mod cardano_immutable_files_full; mod cardano_stake_distribution; mod cardano_transactions; mod interface; mod mithril_stake_distribution; +pub use cardano_database::*; pub use cardano_immutable_files_full::*; pub use cardano_stake_distribution::*; pub use cardano_transactions::*; diff --git a/mithril-aggregator/src/dependency_injection/builder.rs b/mithril-aggregator/src/dependency_injection/builder.rs index d95ecc8f5c0..c09d6fb19ea 100644 --- a/mithril-aggregator/src/dependency_injection/builder.rs +++ b/mithril-aggregator/src/dependency_injection/builder.rs @@ -52,8 +52,9 @@ use mithril_persistence::{ use super::{DependenciesBuilderError, EpochServiceWrapper, Result}; use crate::{ artifact_builder::{ - CardanoImmutableFilesFullArtifactBuilder, CardanoStakeDistributionArtifactBuilder, - CardanoTransactionsArtifactBuilder, MithrilStakeDistributionArtifactBuilder, + CardanoDatabaseArtifactBuilder, CardanoImmutableFilesFullArtifactBuilder, + CardanoStakeDistributionArtifactBuilder, CardanoTransactionsArtifactBuilder, + MithrilStakeDistributionArtifactBuilder, }, configuration::ExecutionEnvironment, database::repository::{ @@ -1208,11 +1209,17 @@ impl DependenciesBuilder { let stake_store = self.get_stake_store().await?; let cardano_stake_distribution_artifact_builder = Arc::new(CardanoStakeDistributionArtifactBuilder::new(stake_store)); + let cardano_database_artifact_builder = Arc::new(CardanoDatabaseArtifactBuilder::new( + self.configuration.db_directory.clone(), + &cardano_node_version, + self.configuration.snapshot_compression_algorithm, + )); let dependencies = SignedEntityServiceArtifactsDependencies::new( mithril_stake_distribution_artifact_builder, cardano_immutable_files_full_artifact_builder, cardano_transactions_artifact_builder, cardano_stake_distribution_artifact_builder, + cardano_database_artifact_builder, ); let signed_entity_service = Arc::new(MithrilSignedEntityService::new( signed_entity_storer, diff --git a/mithril-aggregator/src/services/signed_entity.rs b/mithril-aggregator/src/services/signed_entity.rs index 57b41565f05..83dacd8d385 100644 --- a/mithril-aggregator/src/services/signed_entity.rs +++ b/mithril-aggregator/src/services/signed_entity.rs @@ -11,9 +11,9 @@ use tokio::task::JoinHandle; use mithril_common::{ entities::{ - BlockNumber, CardanoDbBeacon, CardanoStakeDistribution, CardanoTransactionsSnapshot, - Certificate, Epoch, MithrilStakeDistribution, SignedEntity, SignedEntityType, - SignedEntityTypeDiscriminants, Snapshot, + BlockNumber, CardanoDatabaseSnapshot, CardanoDbBeacon, CardanoStakeDistribution, + CardanoTransactionsSnapshot, Certificate, Epoch, MithrilStakeDistribution, SignedEntity, + SignedEntityType, SignedEntityTypeDiscriminants, Snapshot, }, logging::LoggerExtensions, signable_builder::Artifact, @@ -89,6 +89,8 @@ pub struct MithrilSignedEntityService { signed_entity_type_lock: Arc, cardano_stake_distribution_artifact_builder: Arc>, + cardano_database_artifact_builder: + Arc>, metrics_service: Arc, logger: Logger, } @@ -103,6 +105,8 @@ pub struct SignedEntityServiceArtifactsDependencies { Arc>, cardano_stake_distribution_artifact_builder: Arc>, + cardano_database_artifact_builder: + Arc>, } impl SignedEntityServiceArtifactsDependencies { @@ -120,12 +124,16 @@ impl SignedEntityServiceArtifactsDependencies { cardano_stake_distribution_artifact_builder: Arc< dyn ArtifactBuilder, >, + cardano_database_artifact_builder: Arc< + dyn ArtifactBuilder, + >, ) -> Self { Self { mithril_stake_distribution_artifact_builder, cardano_immutable_files_full_artifact_builder, cardano_transactions_artifact_builder, cardano_stake_distribution_artifact_builder, + cardano_database_artifact_builder, } } } @@ -149,6 +157,7 @@ impl MithrilSignedEntityService { .cardano_transactions_artifact_builder, cardano_stake_distribution_artifact_builder: dependencies .cardano_stake_distribution_artifact_builder, + cardano_database_artifact_builder: dependencies.cardano_database_artifact_builder, signed_entity_type_lock, metrics_service, logger: logger.new_with_component_name::(), @@ -247,11 +256,16 @@ impl MithrilSignedEntityService { ) })?, )), - SignedEntityType::CardanoDatabase(_) => { - Err(anyhow::anyhow!( - "Signable builder service can not compute artifact for Cardano database because it is not yet implemented." - )) - } + SignedEntityType::CardanoDatabase(beacon) => Ok(Arc::new( + self.cardano_database_artifact_builder + .compute_artifact(beacon, certificate) + .await + .with_context(|| { + format!( + "Signed Entity Service can not compute artifact for entity type: '{signed_entity_type}'" + ) + })? + )), } } @@ -510,6 +524,8 @@ mod tests { MockArtifactBuilder, mock_cardano_stake_distribution_artifact_builder: MockArtifactBuilder, + mock_cardano_database_artifact_builder: + MockArtifactBuilder, } impl MockDependencyInjector { @@ -532,6 +548,10 @@ mod tests { Epoch, CardanoStakeDistribution, >::new(), + mock_cardano_database_artifact_builder: MockArtifactBuilder::< + CardanoDbBeacon, + CardanoDatabaseSnapshot, + >::new(), } } @@ -541,6 +561,7 @@ mod tests { Arc::new(self.mock_cardano_immutable_files_full_artifact_builder), Arc::new(self.mock_cardano_transactions_artifact_builder), Arc::new(self.mock_cardano_stake_distribution_artifact_builder), + Arc::new(self.mock_cardano_database_artifact_builder), ); MithrilSignedEntityService::new( Arc::new(self.mock_signed_entity_storer), @@ -597,6 +618,7 @@ mod tests { Arc::new(cardano_immutable_files_full_long_artifact_builder), Arc::new(self.mock_cardano_transactions_artifact_builder), Arc::new(self.mock_cardano_stake_distribution_artifact_builder), + Arc::new(self.mock_cardano_database_artifact_builder), ); MithrilSignedEntityService::new( Arc::new(self.mock_signed_entity_storer), @@ -822,17 +844,48 @@ mod tests { } #[tokio::test] - async fn build_cardano_database_artifact_when_given_cardano_database_entity_type_return_error() - { - let mock_container = MockDependencyInjector::new(); + async fn build_cardano_database_artifact_when_given_cardano_database_entity_type() { + let mut mock_container = MockDependencyInjector::new(); + + let cardano_database_expected = fake_data::cardano_database_snapshots(1) + .first() + .unwrap() + .to_owned(); + + mock_container + .mock_cardano_database_artifact_builder + .expect_compute_artifact() + .times(1) + .returning(|_, _| { + Ok(fake_data::cardano_database_snapshots(1) + .first() + .unwrap() + .to_owned()) + }); + let artifact_builder_service = mock_container.build_artifact_builder_service(); + let certificate = fake_data::certificate("hash".to_string()); let signed_entity_type = SignedEntityType::CardanoDatabase(CardanoDbBeacon::default()); - - artifact_builder_service + let artifact = artifact_builder_service .compute_artifact(signed_entity_type.clone(), &certificate) .await - .expect_err("Should return error because CardanoDatabase is not implemented yet."); + .unwrap(); + + assert_expected(&cardano_database_expected, &artifact); + } + + #[tokio::test] + async fn should_store_the_artifact_when_creating_artifact_for_a_cardano_database() { + generic_test_that_the_artifact_is_stored( + SignedEntityType::CardanoDatabase(CardanoDbBeacon::default()), + fake_data::cardano_database_snapshots(1) + .first() + .unwrap() + .to_owned(), + &|mock_injector| &mut mock_injector.mock_cardano_database_artifact_builder, + ) + .await; } async fn generic_test_that_the_artifact_is_stored< diff --git a/mithril-common/Cargo.toml b/mithril-common/Cargo.toml index c74863fa44d..7ca31a7b93f 100644 --- a/mithril-common/Cargo.toml +++ b/mithril-common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-common" -version = "0.4.92" +version = "0.4.93" description = "Common types, interfaces, and utilities for Mithril nodes." authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-common/src/digesters/cardano_immutable_digester.rs b/mithril-common/src/digesters/cardano_immutable_digester.rs index 4fe2f417b83..d27c26d3568 100644 --- a/mithril-common/src/digesters/cardano_immutable_digester.rs +++ b/mithril-common/src/digesters/cardano_immutable_digester.rs @@ -617,7 +617,7 @@ mod tests { let immutable_db = db_builder("hash_computation_is_quicker_with_a_full_cache") .with_immutables(&(1..=50).collect::>()) .append_immutable_trio() - .set_file_size(65536) + .set_immutable_trio_file_size(65538) .build(); let cache = MemoryImmutableFileDigestCacheProvider::default(); let logger = TestLogger::stdout(); diff --git a/mithril-common/src/digesters/dummy_immutable_db_builder.rs b/mithril-common/src/digesters/dummy_immutable_db_builder.rs index 881204d13a8..9e5ca6d750f 100644 --- a/mithril-common/src/digesters/dummy_immutable_db_builder.rs +++ b/mithril-common/src/digesters/dummy_immutable_db_builder.rs @@ -6,13 +6,21 @@ use std::{ path::{Path, PathBuf}, }; +const IMMUTABLE_DIR: &str = "immutable"; +const LEDGER_DIR: &str = "ledger"; +const VOLATILE_DIR: &str = "volatile"; + /// A [DummyImmutableDb] builder. pub struct DummyImmutablesDbBuilder { dir: PathBuf, immutables_to_write: Vec, non_immutables_to_write: Vec, append_uncompleted_trio: bool, - file_size: Option, + immutable_file_size: Option, + ledger_files_to_write: Vec, + ledger_file_size: Option, + volatile_files_to_write: Vec, + volatile_file_size: Option, } /// A dummy cardano immutable db. @@ -47,11 +55,15 @@ impl DummyImmutablesDbBuilder { /// system temp directory, if it exists already it will be cleaned. pub fn new(dir_name: &str) -> Self { Self { - dir: Self::get_test_dir(dir_name), + dir: get_test_dir(dir_name), immutables_to_write: vec![], non_immutables_to_write: vec![], append_uncompleted_trio: false, - file_size: None, + immutable_file_size: None, + ledger_files_to_write: vec![], + ledger_file_size: None, + volatile_files_to_write: vec![], + volatile_file_size: None, } } @@ -68,6 +80,30 @@ impl DummyImmutablesDbBuilder { self } + /// Set ledger files to write to the db in the 'ledger' subdirectory. + pub fn with_ledger_files(&mut self, files: &[&str]) -> &mut Self { + self.ledger_files_to_write = files.iter().map(|name| name.to_string()).collect(); + self + } + + /// Set the size of all ledger files written by [build][Self::build] to the given `file_size` in bytes. + pub fn set_ledger_file_size(&mut self, file_size: u64) -> &mut Self { + self.ledger_file_size = Some(file_size); + self + } + + /// Set volatile files to write to the db in the 'volatile' subdirectory. + pub fn with_volatile_files(&mut self, files: &[&str]) -> &mut Self { + self.volatile_files_to_write = files.iter().map(|f| f.to_string()).collect(); + self + } + + /// Set the size of all volatile files written by [build][Self::build] to the given `file_size` in bytes. + pub fn set_volatile_file_size(&mut self, file_size: u64) -> &mut Self { + self.volatile_file_size = Some(file_size); + self + } + /// Makes [build][Self::build] add another trio of immutables file, that won't be included /// in its returned vec, to simulate the last 3 'uncompleted / wip' files that can be found in /// a cardano immutable db. @@ -76,11 +112,16 @@ impl DummyImmutablesDbBuilder { self } - /// Set the size of all files written by [build][Self::build] to the given `file_size` in bytes. + /// Set the size of all immutable files written by [build][Self::build] to the given `file_size` in bytes. /// /// Note: by default the size of the produced files is less than a 1kb. - pub fn set_file_size(&mut self, file_size: u64) -> &mut Self { - self.file_size = Some(file_size); + pub fn set_immutable_trio_file_size(&mut self, trio_file_size: u64) -> &mut Self { + assert!( + trio_file_size % 3 == 0, + "'trio_file_size' must be a multiple of 3" + ); + + self.immutable_file_size = Some(trio_file_size / 3); self } @@ -92,8 +133,8 @@ impl DummyImmutablesDbBuilder { if self.append_uncompleted_trio { write_immutable_trio( - self.file_size, - &self.dir, + self.immutable_file_size, + &self.dir.join(IMMUTABLE_DIR), match immutable_numbers.last() { None => 0, Some(last) => last + 1, @@ -102,22 +143,40 @@ impl DummyImmutablesDbBuilder { } for non_immutable in &self.non_immutables_to_write { - non_immutables_files.push(write_dummy_file(self.file_size, &self.dir, non_immutable)); + non_immutables_files.push(write_dummy_file( + self.immutable_file_size, + &self.dir.join(IMMUTABLE_DIR), + non_immutable, + )); + } + + for filename in &self.ledger_files_to_write { + write_dummy_file(self.ledger_file_size, &self.dir.join(LEDGER_DIR), filename); + } + + for filename in &self.volatile_files_to_write { + write_dummy_file( + self.volatile_file_size, + &self.dir.join(VOLATILE_DIR), + filename, + ); } DummyImmutableDb { - dir: self.dir.clone(), + dir: self.dir.join(IMMUTABLE_DIR), immutables_files: immutable_numbers .into_iter() - .flat_map(|ifn| write_immutable_trio(self.file_size, &self.dir, ifn)) + .flat_map(|ifn| { + write_immutable_trio( + self.immutable_file_size, + &self.dir.join(IMMUTABLE_DIR), + ifn, + ) + }) .collect::>(), non_immutables_files, } } - - fn get_test_dir(subdir_name: &str) -> PathBuf { - TempDir::create(subdir_name, "immutable") - } } fn write_immutable_trio( @@ -156,3 +215,12 @@ fn write_dummy_file(optional_size: Option, dir: &Path, filename: &str) -> P file } + +fn get_test_dir(subdir_name: &str) -> PathBuf { + let db_dir = TempDir::create("test_cardano_db", subdir_name); + for subdir_name in [LEDGER_DIR, IMMUTABLE_DIR, VOLATILE_DIR] { + std::fs::create_dir(db_dir.join(subdir_name)).unwrap(); + } + + db_dir +} diff --git a/mithril-common/src/entities/cardano_database.rs b/mithril-common/src/entities/cardano_database.rs new file mode 100644 index 00000000000..9c6b5f0284e --- /dev/null +++ b/mithril-common/src/entities/cardano_database.rs @@ -0,0 +1,89 @@ +use semver::Version; +use serde::{Deserialize, Serialize}; + +use crate::{ + entities::{CardanoDbBeacon, CompressionAlgorithm}, + signable_builder::Artifact, +}; + +/// Cardano database snapshot. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CardanoDatabaseSnapshot { + /// Merkle root of the Cardano database snapshot. + pub merkle_root: String, + + /// Mithril beacon on the Cardano chain. + pub beacon: CardanoDbBeacon, + + /// Size of the uncompressed Cardano database files. + pub total_db_size_uncompressed: u64, + + /// Locations of the Cardano database artifacts. + pub locations: ArtifactsLocations, + + /// Compression algorithm of the Cardano database artifacts. + pub compression_algorithm: CompressionAlgorithm, + + /// Version of the Cardano node used to create the snapshot. + pub cardano_node_version: String, +} + +impl CardanoDatabaseSnapshot { + /// [CardanoDatabaseSnapshot] factory + pub fn new( + merkle_root: String, + beacon: CardanoDbBeacon, + total_db_size_uncompressed: u64, + locations: ArtifactsLocations, + compression_algorithm: CompressionAlgorithm, + cardano_node_version: &Version, + ) -> Self { + let cardano_node_version = format!("{cardano_node_version}"); + + Self { + merkle_root, + beacon, + locations, + total_db_size_uncompressed, + compression_algorithm, + cardano_node_version, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "type")] +enum DigestLocation { + Aggregator { uri: String }, + CloudStorage { uri: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "type")] +enum ImmutablesLocation { + CloudStorage { uri: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "type")] +enum AncillaryLocation { + CloudStorage { uri: String }, +} + +/// Locations of the Cardano database related files. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArtifactsLocations { + /// Locations of the the immutable file digests. + digest: Vec, + /// Locations of the immutable files. + immutables: Vec, + /// Locations of the ancillary files. + ancillary: Vec, +} + +#[typetag::serde] +impl Artifact for CardanoDatabaseSnapshot { + fn get_id(&self) -> String { + self.merkle_root.clone() + } +} diff --git a/mithril-common/src/entities/compression_algorithm.rs b/mithril-common/src/entities/compression_algorithm.rs new file mode 100644 index 00000000000..b35cca04ad4 --- /dev/null +++ b/mithril-common/src/entities/compression_algorithm.rs @@ -0,0 +1,39 @@ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIter, IntoEnumIterator}; + +/// Compression algorithm for the snapshot archive artifacts. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default, EnumIter, Display)] +#[serde(rename_all = "lowercase")] +pub enum CompressionAlgorithm { + /// Gzip compression format + #[default] + Gzip, + /// Zstandard compression format + Zstandard, +} + +impl CompressionAlgorithm { + /// Get the extension associated to tar archive using the current algorithm. + pub fn tar_file_extension(&self) -> String { + match self { + CompressionAlgorithm::Gzip => "tar.gz".to_owned(), + CompressionAlgorithm::Zstandard => "tar.zst".to_owned(), + } + } + + /// List all the available [algorithms][CompressionAlgorithm]. + pub fn list() -> Vec { + Self::iter().collect() + } + + /// Those ratio will be multiplied by the snapshot size to check if the available + /// disk space is sufficient to store the archive plus the extracted files. + /// If the available space is lower than that, a warning is raised. + /// Those ratio have been experimentally established. + pub fn free_space_snapshot_ratio(&self) -> f64 { + match self { + CompressionAlgorithm::Gzip => 2.5, + CompressionAlgorithm::Zstandard => 4.0, + } + } +} diff --git a/mithril-common/src/entities/mod.rs b/mithril-common/src/entities/mod.rs index a8dbfc24c57..ee48fb0e0dd 100644 --- a/mithril-common/src/entities/mod.rs +++ b/mithril-common/src/entities/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod arithmetic_operation_wrapper; mod block_number; mod block_range; mod cardano_chain_point; +mod cardano_database; mod cardano_db_beacon; mod cardano_network; mod cardano_stake_distribution; @@ -13,6 +14,7 @@ mod cardano_transactions_snapshot; mod certificate; mod certificate_metadata; mod certificate_pending; +mod compression_algorithm; mod epoch; mod http_server_error; mod mithril_stake_distribution; @@ -31,6 +33,7 @@ mod type_alias; pub use block_number::BlockNumber; pub use block_range::{BlockRange, BlockRangeLength, BlockRangesSequence}; pub use cardano_chain_point::{BlockHash, ChainPoint}; +pub use cardano_database::{ArtifactsLocations, CardanoDatabaseSnapshot}; pub use cardano_db_beacon::CardanoDbBeacon; pub use cardano_network::CardanoNetwork; pub use cardano_stake_distribution::CardanoStakeDistribution; @@ -40,6 +43,7 @@ pub use cardano_transactions_snapshot::CardanoTransactionsSnapshot; pub use certificate::{Certificate, CertificateSignature}; pub use certificate_metadata::{CertificateMetadata, StakeDistributionParty}; pub use certificate_pending::CertificatePending; +pub use compression_algorithm::*; pub use epoch::{Epoch, EpochError}; pub use http_server_error::{ClientError, ServerError}; pub use mithril_stake_distribution::MithrilStakeDistribution; @@ -51,6 +55,6 @@ pub use signed_entity_type::*; pub use signer::{Signer, SignerWithStake}; pub use single_signatures::*; pub use slot_number::SlotNumber; -pub use snapshot::{CompressionAlgorithm, Snapshot}; +pub use snapshot::Snapshot; pub use time_point::*; pub use type_alias::*; diff --git a/mithril-common/src/entities/snapshot.rs b/mithril-common/src/entities/snapshot.rs index 5e1e73a8cc1..9a4f7ad91f4 100644 --- a/mithril-common/src/entities/snapshot.rs +++ b/mithril-common/src/entities/snapshot.rs @@ -1,7 +1,9 @@ -use crate::{entities::CardanoDbBeacon, signable_builder::Artifact}; +use crate::{ + entities::{CardanoDbBeacon, CompressionAlgorithm}, + signable_builder::Artifact, +}; use semver::Version; use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIter, IntoEnumIterator}; /// Snapshot represents a snapshot file and its metadata #[derive(Clone, Debug, PartialEq, Eq, Default, Serialize, Deserialize)] @@ -28,43 +30,6 @@ pub struct Snapshot { pub cardano_node_version: String, } -/// Compression algorithm for the snapshot archive artifacts. -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default, EnumIter, Display)] -#[serde(rename_all = "lowercase")] -pub enum CompressionAlgorithm { - /// Gzip compression format - #[default] - Gzip, - /// Zstandard compression format - Zstandard, -} - -impl CompressionAlgorithm { - /// Get the extension associated to tar archive using the current algorithm. - pub fn tar_file_extension(&self) -> String { - match self { - CompressionAlgorithm::Gzip => "tar.gz".to_owned(), - CompressionAlgorithm::Zstandard => "tar.zst".to_owned(), - } - } - - /// List all the available [algorithms][CompressionAlgorithm]. - pub fn list() -> Vec { - Self::iter().collect() - } - - /// Those ratio will be multiplied by the snapshot size to check if the available - /// disk space is sufficient to store the archive plus the extracted files. - /// If the available space is lower than that, a warning is raised. - /// Those ratio have been experimentally established. - pub fn free_space_snapshot_ratio(&self) -> f64 { - match self { - CompressionAlgorithm::Gzip => 2.5, - CompressionAlgorithm::Zstandard => 4.0, - } - } -} - impl Snapshot { /// Snapshot factory pub fn new>( diff --git a/mithril-common/src/test_utils/fake_data.rs b/mithril-common/src/test_utils/fake_data.rs index 25fc0a312ee..a976b4e1bac 100644 --- a/mithril-common/src/test_utils/fake_data.rs +++ b/mithril-common/src/test_utils/fake_data.rs @@ -5,9 +5,9 @@ use semver::Version; use crate::crypto_helper::{self, ProtocolMultiSignature}; use crate::entities::{ - self, BlockNumber, CertificateMetadata, CertificateSignature, CompressionAlgorithm, Epoch, - LotteryIndex, ProtocolMessage, ProtocolMessagePartKey, SignedEntityType, SingleSignatures, - SlotNumber, StakeDistribution, StakeDistributionParty, + self, ArtifactsLocations, BlockNumber, CertificateMetadata, CertificateSignature, + CompressionAlgorithm, Epoch, LotteryIndex, ProtocolMessage, ProtocolMessagePartKey, + SignedEntityType, SingleSignatures, SlotNumber, StakeDistribution, StakeDistributionParty, }; use crate::test_utils::MithrilFixtureBuilder; @@ -277,3 +277,26 @@ pub fn cardano_stake_distribution(epoch: Epoch) -> entities::CardanoStakeDistrib stake_distribution, } } + +/// Fake Cardano Database snapshots +pub fn cardano_database_snapshots(total: u64) -> Vec { + (1..total + 1) + .map(|cardano_database_id| { + let merkle_root = format!("1{cardano_database_id}").repeat(20); + let mut beacon = beacon(); + beacon.immutable_file_number += cardano_database_id; + let total_db_size_uncompressed = cardano_database_id * 100000; + let cardano_node_version = Version::parse("1.0.0").unwrap(); + let locations = ArtifactsLocations::default(); + + entities::CardanoDatabaseSnapshot::new( + merkle_root, + beacon, + total_db_size_uncompressed, + locations, + CompressionAlgorithm::Gzip, + &cardano_node_version, + ) + }) + .collect::>() +}