diff --git a/grovedb-version/src/lib.rs b/grovedb-version/src/lib.rs index 348150053..805b33a27 100644 --- a/grovedb-version/src/lib.rs +++ b/grovedb-version/src/lib.rs @@ -72,6 +72,31 @@ macro_rules! check_grovedb_v0_or_v1 { }}; } +#[macro_export] +macro_rules! check_grovedb_v0_v1_or_v2 { + ($method:expr, $version:expr) => {{ + const EXPECTED_VERSION_V0: u16 = 0; + const EXPECTED_VERSION_V1: u16 = 1; + const EXPECTED_VERSION_V2: u16 = 2; + if $version != EXPECTED_VERSION_V0 + && $version != EXPECTED_VERSION_V1 + && $version != EXPECTED_VERSION_V2 + { + return Err($crate::error::GroveVersionError::UnknownVersionMismatch { + method: $method.to_string(), + known_versions: vec![ + EXPECTED_VERSION_V0, + EXPECTED_VERSION_V1, + EXPECTED_VERSION_V2, + ], + received: $version, + } + .into()); + } + $version + }}; +} + #[macro_export] macro_rules! check_merk_v0_with_cost { ($method:expr, $version:expr) => {{ diff --git a/grovedb-version/src/version/merk_versions.rs b/grovedb-version/src/version/merk_versions.rs index f0524ab36..f09826468 100644 --- a/grovedb-version/src/version/merk_versions.rs +++ b/grovedb-version/src/version/merk_versions.rs @@ -18,6 +18,13 @@ pub struct MerkBatchVersions { pub struct MerkAverageCaseCostsVersions { pub add_average_case_merk_propagate: FeatureVersion, pub sum_tree_estimated_size: FeatureVersion, + /// `EstimatedLayerSizes::value_with_feature_and_flags_size`. Version + /// 0 keeps the pre-existing **unweighted** Mix average + /// `Σ size_i / Σ weight_i` (used by grove v1/v2, consensus-locked). + /// Version 1 switches to a proper weighted average + /// `Σ (size_i · weight_i) / Σ weight_i`. Output for non-Mix + /// variants is identical across versions. + pub value_with_feature_and_flags_size: FeatureVersion, } /// Merk-level proof method versions. diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index ad7b882d5..3c6f9ee34 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -216,6 +216,7 @@ pub const GROVE_V1: GroveVersion = GroveVersion { average_case_costs: MerkAverageCaseCostsVersions { add_average_case_merk_propagate: 0, sum_tree_estimated_size: 0, + value_with_feature_and_flags_size: 0, }, // `prove_count_offset_on_range` is implementation-version 0 // here too — but in grove v1 the V0 proof envelope rejects diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 959bfa0f2..ea40d40ed 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -216,6 +216,9 @@ pub const GROVE_V2: GroveVersion = GroveVersion { average_case_costs: MerkAverageCaseCostsVersions { add_average_case_merk_propagate: 1, // changed sum_tree_estimated_size: 1, // changed + // grove v2 is consensus-locked to the legacy unweighted + // Mix average; v3 bumps this to 1 with the fixed formula. + value_with_feature_and_flags_size: 0, }, // See the comment in v1.rs — `prove_count_offset_on_range` is // not reachable from v2's prove path (V0 envelope rejects diff --git a/grovedb-version/src/version/v3.rs b/grovedb-version/src/version/v3.rs index 10a32fb3a..2624e5acd 100644 --- a/grovedb-version/src/version/v3.rs +++ b/grovedb-version/src/version/v3.rs @@ -214,8 +214,24 @@ pub const GROVE_V3: GroveVersion = GroveVersion { merk_versions: MerkVersions { batch: MerkBatchVersions { commit: 1 }, // return accumulated batch costs average_case_costs: MerkAverageCaseCostsVersions { - add_average_case_merk_propagate: 1, - sum_tree_estimated_size: 1, + // Bumped from 1 → 2: v2 fixes the Mix-arm divisor on both + // `replaced_bytes` and `storage_loaded_bytes`. v1 computed + // `Σ(weight_i · cost_i) / (nodes_updated · total_weight)` + // (off by a factor of `nodes_updated²` vs the equivalent + // homogeneous-layer cost); v2 computes + // `nodes_updated · Σ(weight_i · cost_i) / total_weight`. + add_average_case_merk_propagate: 2, + // Bumped from 1 → 2: v2 folds the four `Provable*` tree-type + // weight fields on `EstimatedSumTrees::SomeSumTrees` into the + // weighted-average formula. v0/v1 outputs are preserved for + // already-shipped grove versions (v1 → v0 formula, v2 → v1 + // formula). + sum_tree_estimated_size: 2, + // Bumped from 0 → 1: v1 fixes the Mix-arm "average" to a + // proper weighted average. v0 returned + // `Σ size_i / Σ weight_i` (low-biased and ignored weights); + // v1 returns `Σ (size_i · weight_i) / Σ weight_i`. + value_with_feature_and_flags_size: 1, }, proof: MerkProofVersions { // Initial implementation; introduced alongside the V1 diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index edbfd8076..b29110b4c 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -962,6 +962,10 @@ mod tests { count_trees_weight: 0, count_sum_trees_weight: 0, non_sum_trees_weight: 1, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, }, None, ), diff --git a/grovedb/src/tests/misc_coverage_tests.rs b/grovedb/src/tests/misc_coverage_tests.rs index 52b401be5..f2197c504 100644 --- a/grovedb/src/tests/misc_coverage_tests.rs +++ b/grovedb/src/tests/misc_coverage_tests.rs @@ -1414,6 +1414,10 @@ fn batch_average_case_sum_tree_insert() { count_trees_weight: 0, count_sum_trees_weight: 0, non_sum_trees_weight: 1, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, }, None, ), @@ -2008,6 +2012,10 @@ fn batch_average_case_replace_sum_item_cost() { count_trees_weight: 0, count_sum_trees_weight: 0, non_sum_trees_weight: 1, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, }, None, ), @@ -2060,6 +2068,10 @@ fn batch_average_case_replace_sum_tree_cost() { count_trees_weight: 0, count_sum_trees_weight: 0, non_sum_trees_weight: 1, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, }, None, ), @@ -2078,6 +2090,10 @@ fn batch_average_case_replace_sum_tree_cost() { count_trees_weight: 0, count_sum_trees_weight: 0, non_sum_trees_weight: 1, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, }, None, ), @@ -2124,6 +2140,10 @@ fn batch_average_case_insert_into_sum_tree() { count_trees_weight: 0, count_sum_trees_weight: 0, non_sum_trees_weight: 1, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, }, None, ), @@ -2416,6 +2436,10 @@ fn batch_average_case_delete_sum_tree_cost() { count_trees_weight: 0, count_sum_trees_weight: 0, non_sum_trees_weight: 1, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, }, None, ), @@ -2434,6 +2458,10 @@ fn batch_average_case_delete_sum_tree_cost() { count_trees_weight: 0, count_sum_trees_weight: 0, non_sum_trees_weight: 1, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, }, None, ), @@ -2862,6 +2890,10 @@ fn batch_average_case_insert_count_tree_cost() { count_trees_weight: 1, count_sum_trees_weight: 0, non_sum_trees_weight: 1, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, }, None, ), @@ -3412,6 +3444,10 @@ fn batch_average_case_insert_big_sum_tree() { count_trees_weight: 0, count_sum_trees_weight: 0, non_sum_trees_weight: 1, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, }, None, ), diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs deleted file mode 100644 index cccd4b333..000000000 --- a/merk/src/estimated_costs/average_case_costs.rs +++ /dev/null @@ -1,1476 +0,0 @@ -//! Average case costs for Merk - -#[cfg(feature = "minimal")] -use grovedb_costs::{CostResult, CostsExt, OperationCost}; -#[cfg(feature = "minimal")] -use grovedb_version::{check_grovedb_v0_or_v1, error::GroveVersionError, version::GroveVersion}; -#[cfg(feature = "minimal")] -use integer_encoding::VarInt; - -#[cfg(feature = "minimal")] -use crate::{ - error::Error, - estimated_costs::LAYER_COST_SIZE, - tree::{kv::KV, Link, TreeNode}, - HASH_BLOCK_SIZE, HASH_BLOCK_SIZE_U32, HASH_LENGTH, HASH_LENGTH_U32, -}; -use crate::{merk::NodeType, tree_type::TreeType}; - -#[cfg(feature = "minimal")] -/// Average key size -pub type AverageKeySize = u8; -#[cfg(feature = "minimal")] -/// Average value size -pub type AverageValueSize = u32; -#[cfg(feature = "minimal")] -/// Average flags size -pub type AverageFlagsSize = u32; -#[cfg(feature = "minimal")] -/// Weight -pub type Weight = u8; - -#[cfg(feature = "minimal")] -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -/// Estimated number of sum trees -#[derive(Default)] -pub enum EstimatedSumTrees { - /// No sum trees - #[default] - NoSumTrees, - /// Some sum trees - SomeSumTrees { - /// Sum trees weight - sum_trees_weight: Weight, - /// Big Sum trees weight - big_sum_trees_weight: Weight, - /// Count trees weight - count_trees_weight: Weight, - /// Count Sum trees weight - count_sum_trees_weight: Weight, - /// Non sum trees weight - non_sum_trees_weight: Weight, - }, - /// All sum trees - AllSumTrees, - /// All big sum trees - AllBigSumTrees, - /// All count trees - AllCountTrees, - /// All count sum trees - AllCountSumTrees, -} - -#[cfg(feature = "minimal")] -impl EstimatedSumTrees { - fn estimated_size(&self, grove_version: &GroveVersion) -> Result { - let version = check_grovedb_v0_or_v1!( - "EstimatedSumTrees::estimated_size", - grove_version - .merk_versions - .average_case_costs - .sum_tree_estimated_size - ); - match self { - EstimatedSumTrees::NoSumTrees => Ok(0), - EstimatedSumTrees::SomeSumTrees { - sum_trees_weight, - big_sum_trees_weight, - count_trees_weight, - count_sum_trees_weight, - non_sum_trees_weight, - } => { - // Example calculation including new weights - let total_weight = *sum_trees_weight as u32 - + *big_sum_trees_weight as u32 - + *count_trees_weight as u32 - + *count_sum_trees_weight as u32 - + *non_sum_trees_weight as u32; - if total_weight == 0 { - return Err(Error::DivideByZero("weights add up to 0")); - }; - if version == 0 { - Ok((*non_sum_trees_weight as u32 * 9) - / (*sum_trees_weight as u32 + *non_sum_trees_weight as u32)) - } else if version == 1 { - let estimated_size = (*sum_trees_weight as u32 - * TreeType::SumTree.inner_node_type().cost()) - .checked_add( - *big_sum_trees_weight as u32 - * TreeType::BigSumTree.inner_node_type().cost(), - ) - .and_then(|sum| { - sum.checked_add( - *count_trees_weight as u32 - * TreeType::CountTree.inner_node_type().cost(), - ) - }) - .and_then(|sum| { - sum.checked_add( - *count_sum_trees_weight as u32 - * TreeType::CountSumTree.inner_node_type().cost(), - ) - }) - .ok_or(Error::Overflow("Estimated size calculation overflowed"))?; - - Ok(estimated_size / total_weight) - } else { - Err(Error::CorruptedCodeExecution("we already checked versions")) - } - } - EstimatedSumTrees::AllSumTrees => Ok(TreeType::SumTree.inner_node_type().cost()), - EstimatedSumTrees::AllBigSumTrees => Ok(TreeType::BigSumTree.inner_node_type().cost()), - EstimatedSumTrees::AllCountTrees => Ok(TreeType::CountTree.inner_node_type().cost()), - EstimatedSumTrees::AllCountSumTrees => { - Ok(TreeType::CountSumTree.inner_node_type().cost()) - } - } - } -} - -#[cfg(feature = "minimal")] -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -/// Estimated layer sizes -pub enum EstimatedLayerSizes { - /// All subtrees - AllSubtrees(AverageKeySize, EstimatedSumTrees, Option), - /// All items - AllItems(AverageKeySize, AverageValueSize, Option), - /// References - AllReference(AverageKeySize, AverageValueSize, Option), - /// Mix - Mix { - /// Subtrees size - subtrees_size: Option<( - AverageKeySize, - EstimatedSumTrees, - Option, - Weight, - )>, - /// Items size - items_size: Option<( - AverageKeySize, - AverageValueSize, - Option, - Weight, - )>, - /// References size - references_size: Option<( - AverageKeySize, - AverageValueSize, - Option, - Weight, - )>, - }, -} - -#[cfg(feature = "minimal")] -impl EstimatedLayerSizes { - /// Return average flags size for layer - pub fn layered_flags_size(&self) -> Result<&Option, Error> { - match self { - EstimatedLayerSizes::AllSubtrees(_, _, flags_size) => Ok(flags_size), - EstimatedLayerSizes::Mix { - subtrees_size: subtree_size, - items_size: _, - references_size: _, - } => { - if let Some((_, _, flags_size, _)) = subtree_size { - Ok(flags_size) - } else { - Err(Error::WrongEstimatedCostsElementTypeForLevel( - "this mixed layer does not have costs for trees", - )) - } - } - _ => Err(Error::WrongEstimatedCostsElementTypeForLevel( - "this layer does not have costs for trees", - )), - } - } - - /// Returns the size of a subtree's feature and flags - /// This only takes into account subtrees in the estimated layer info - /// Only should be used when it is known to be a subtree - pub fn subtree_with_feature_and_flags_size( - &self, - grove_version: &GroveVersion, - ) -> Result { - match self { - EstimatedLayerSizes::AllSubtrees(_, estimated_sum_trees, flags_size) => { - // 1 for enum type - // 1 for empty - // 1 for flags size - Ok(estimated_sum_trees.estimated_size(grove_version)? - + flags_size.unwrap_or_default() - + 3) - } - EstimatedLayerSizes::Mix { subtrees_size, .. } => match subtrees_size { - None => Err(Error::WrongEstimatedCostsElementTypeForLevel( - "this layer is a mix but doesn't have subtrees", - )), - Some((_, est, fs, _)) => { - Ok(est.estimated_size(grove_version)? + fs.unwrap_or_default() + 3) - } - }, - _ => Err(Error::WrongEstimatedCostsElementTypeForLevel( - "this layer needs to have trees", - )), - } - } - - /// Returns the size of a value's feature and flags - pub fn value_with_feature_and_flags_size( - &self, - grove_version: &GroveVersion, - ) -> Result { - match self { - EstimatedLayerSizes::AllItems(_, average_value_size, flags_size) => { - // 1 for enum type - // 1 for value size - // 1 for flags size - Ok(*average_value_size + flags_size.unwrap_or_default() + 3) - } - EstimatedLayerSizes::AllReference(_, average_value_size, flags_size) => { - // 1 for enum type - // 1 for value size - // 1 for flags size - // 2 for reference hops - Ok(*average_value_size + flags_size.unwrap_or_default() + 5) - } - EstimatedLayerSizes::AllSubtrees(_, estimated_sum_trees, flags_size) => { - // 1 for enum type - // 1 for empty - // 1 for flags size - Ok(estimated_sum_trees.estimated_size(grove_version)? - + flags_size.unwrap_or_default() - + 3) - } - EstimatedLayerSizes::Mix { - subtrees_size, - items_size, - references_size, - } => { - let (item_size, item_weight) = items_size - .as_ref() - .map(|(_, vs, fs, weight)| (vs + fs.unwrap_or_default() + 3, *weight as u32)) - .unwrap_or_default(); - - let (ref_size, ref_weight) = references_size - .as_ref() - .map(|(_, vs, fs, weight)| (vs + fs.unwrap_or_default() + 5, *weight as u32)) - .unwrap_or_default(); - - let (subtree_size, subtree_weight) = match subtrees_size { - None => None, - Some((_, est, fs, weight)) => Some(( - est.estimated_size(grove_version)? + fs.unwrap_or_default() + 3, - *weight as u32, - )), - } - .unwrap_or_default(); - - if item_weight == 0 && ref_weight == 0 && subtree_weight == 0 { - return Err(Error::WrongEstimatedCostsElementTypeForLevel( - "this layer is a mix and does not have items, refs or trees", - )); - } - if item_weight == 0 && ref_weight == 0 { - return Ok(subtree_size); - } - if item_weight == 0 && subtree_weight == 0 { - return Ok(ref_size); - } - if ref_weight == 0 && subtree_weight == 0 { - return Ok(item_size); - } - let combined_weight = item_weight - .checked_add(ref_weight) - .and_then(|a| a.checked_add(subtree_weight)) - .ok_or(Error::Overflow("overflow for value size combining weights"))?; - item_size - .checked_add(ref_size) - .and_then(|a| a.checked_add(subtree_size)) - .and_then(|a| a.checked_div(combined_weight)) - .ok_or(Error::Overflow("overflow for value size")) - } - } - } -} - -#[cfg(feature = "minimal")] -/// Approximate element count -pub type ApproximateElementCount = u32; -#[cfg(feature = "minimal")] -/// Estimated level number -pub type EstimatedLevelNumber = u32; -#[cfg(feature = "minimal")] -/// Estimated to be empty -pub type EstimatedToBeEmpty = bool; - -#[cfg(feature = "minimal")] -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -/// Information on an estimated layer -pub struct EstimatedLayerInformation { - /// The kind of tree we are in - pub tree_type: TreeType, - /// Estimated layer count - pub estimated_layer_count: EstimatedLayerCount, - /// Estimated layer sizes - pub estimated_layer_sizes: EstimatedLayerSizes, -} - -#[cfg(feature = "minimal")] -impl EstimatedLayerInformation {} - -#[cfg(feature = "minimal")] -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -/// Estimated elements and level number of a layer -pub enum EstimatedLayerCount { - /// Potentially at max elements - PotentiallyAtMaxElements, - /// Approximate elements - ApproximateElements(ApproximateElementCount), - /// Estimated level - EstimatedLevel(EstimatedLevelNumber, EstimatedToBeEmpty), -} - -#[cfg(feature = "minimal")] -impl EstimatedLayerCount { - /// Returns true if the tree is estimated to be empty. - pub fn estimated_to_be_empty(&self) -> bool { - match self { - EstimatedLayerCount::ApproximateElements(count) => *count == 0, - EstimatedLayerCount::PotentiallyAtMaxElements => false, - EstimatedLayerCount::EstimatedLevel(_, empty) => *empty, - } - } - - /// Estimate the number of levels based on the size of the tree, for big - /// trees this is very inaccurate. - pub fn estimate_levels(&self) -> u32 { - match self { - EstimatedLayerCount::ApproximateElements(n) => { - if *n == u32::MAX { - 32 - } else { - ((n + 1) as f32).log2().ceil() as u32 - } - } - EstimatedLayerCount::PotentiallyAtMaxElements => 32, - EstimatedLayerCount::EstimatedLevel(n, _) => *n, - } - } -} - -#[cfg(feature = "minimal")] -impl TreeNode { - /// Return estimate of average encoded tree size - pub fn average_case_encoded_tree_size( - not_prefixed_key_len: u32, - estimated_element_size: u32, - node_type: NodeType, - ) -> u32 { - // two option values for the left and right link - // the actual left and right link encoding size - // the encoded kv node size - 2 + (2 * Link::encoded_link_size(not_prefixed_key_len, node_type)) - + KV::encoded_kv_node_size(estimated_element_size, node_type) - } -} - -#[cfg(feature = "minimal")] -/// Add worst case for getting a merk node -pub fn add_average_case_get_merk_node( - cost: &mut OperationCost, - not_prefixed_key_len: u32, - approximate_element_size: u32, - node_type: NodeType, -) -> Result<(), Error> { - // Worst case scenario, the element is not already in memory. - // One direct seek has to be performed to read the node from storage. - cost.seek_count += 1; - - // To write a node to disk, the left link, right link and kv nodes are encoded. - // worst case, the node has both the left and right link present. - cost.storage_loaded_bytes += TreeNode::average_case_encoded_tree_size( - not_prefixed_key_len, - approximate_element_size, - node_type, - ) as u64; - Ok(()) -} - -#[cfg(feature = "minimal")] -/// Add worst case for getting a merk tree -pub fn add_average_case_merk_has_value( - cost: &mut OperationCost, - not_prefixed_key_len: u32, - estimated_element_size: u32, -) { - cost.seek_count += 1; - cost.storage_loaded_bytes += (not_prefixed_key_len + estimated_element_size) as u64; -} - -#[cfg(feature = "minimal")] -/// Add worst case for insertion into merk -pub fn add_average_case_merk_replace_layered( - cost: &mut OperationCost, - key_len: u32, - value_len: u32, - node_type: NodeType, -) { - cost.seek_count += 1; - cost.storage_cost.replaced_bytes = - KV::layered_value_byte_cost_size_for_key_and_value_lengths(key_len, value_len, node_type); - - // first lets add the value hash - cost.hash_node_calls += 1 + ((value_len - 1) / HASH_BLOCK_SIZE_U32); - // then let's add the combine hash - cost.hash_node_calls += 1; - // then let's add the kv_digest_to_kv_hash hash call - let hashed_size = key_len.encode_var_vec().len() as u32 + key_len + HASH_LENGTH_U32; - cost.hash_node_calls += 1 + ((hashed_size - 1) / HASH_BLOCK_SIZE_U32); - // then let's add the two block hashes for the node hash call - cost.hash_node_calls += 2; -} - -#[cfg(feature = "minimal")] -/// Add average case for deletion from merk -pub fn add_average_case_merk_delete_layered( - cost: &mut OperationCost, - _key_len: u32, - value_len: u32, -) { - // todo: verify this - cost.seek_count += 1; - cost.hash_node_calls += 1 + ((value_len - 1) / HASH_BLOCK_SIZE_U32); -} - -#[cfg(feature = "minimal")] -/// Add average case for deletion from merk -pub fn add_average_case_merk_delete(cost: &mut OperationCost, _key_len: u32, value_len: u32) { - // todo: verify this - cost.seek_count += 1; - cost.hash_node_calls += 1 + ((value_len - 1) / HASH_BLOCK_SIZE_U32); -} - -#[cfg(feature = "minimal")] -const fn node_hash_update_count() -> u32 { - // It's a hash of node hash, left and right - let bytes = HASH_LENGTH * 3; - // todo: verify this - - 1 + ((bytes - 1) / HASH_BLOCK_SIZE) as u32 -} - -#[cfg(feature = "minimal")] -/// Add worst case for getting a merk tree root hash -pub fn add_average_case_merk_root_hash(cost: &mut OperationCost) { - cost.hash_node_calls += node_hash_update_count(); -} - -#[cfg(feature = "minimal")] -/// Average case cost of propagating a merk -pub fn average_case_merk_propagate( - input: &EstimatedLayerInformation, - grove_version: &GroveVersion, -) -> CostResult<(), Error> { - let mut cost = OperationCost::default(); - add_average_case_merk_propagate(&mut cost, input, grove_version).wrap_with_cost(cost) -} - -#[cfg(feature = "minimal")] -/// Add average case cost for propagating a merk -pub fn add_average_case_merk_propagate( - cost: &mut OperationCost, - input: &EstimatedLayerInformation, - grove_version: &GroveVersion, -) -> Result<(), Error> { - match grove_version - .merk_versions - .average_case_costs - .add_average_case_merk_propagate - { - 0 => add_average_case_merk_propagate_v0(cost, input, grove_version), - 1 => add_average_case_merk_propagate_v1(cost, input, grove_version), - version => Err(Error::VersionError( - GroveVersionError::UnknownVersionMismatch { - method: "add_average_case_merk_propagate".to_string(), - known_versions: vec![0, 1], - received: version, - }, - )), - } -} -#[cfg(feature = "minimal")] -/// Add average case cost for propagating a merk -fn add_average_case_merk_propagate_v1( - cost: &mut OperationCost, - input: &EstimatedLayerInformation, - grove_version: &GroveVersion, -) -> Result<(), Error> { - let mut nodes_updated = 0; - // Propagation requires to recompute and write hashes up to the root - let EstimatedLayerInformation { - tree_type, - estimated_layer_count, - estimated_layer_sizes, - } = input; - let levels = estimated_layer_count.estimate_levels(); - nodes_updated += levels; - - if levels > 1 { - // we can get about 1 rotation, if there are more than 2 levels - nodes_updated += 1; - } - cost.seek_count += nodes_updated; - - cost.hash_node_calls += nodes_updated * 2; - - cost.storage_cost.replaced_bytes += match estimated_layer_sizes { - EstimatedLayerSizes::AllSubtrees( - average_key_size, - estimated_sum_trees, - average_flags_size, - ) => { - // it is normal to have LAYER_COST_SIZE here, as we add estimated sum tree - // additions right after - let value_len = LAYER_COST_SIZE - + average_flags_size - .map_or(0, |flags_len| flags_len + flags_len.required_space() as u32); - // in order to simplify calculations we get the estimated size and remove the - // cost for the basic merk - let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; - nodes_updated - * (KV::layered_value_byte_cost_size_for_key_and_value_lengths( - *average_key_size as u32, - value_len, - tree_type.inner_node_type(), - ) + sum_tree_addition) - } - EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) - | EstimatedLayerSizes::AllReference( - average_key_size, - average_item_size, - average_flags_size, - ) => { - let flags_len = average_flags_size.unwrap_or(0); - let average_value_len = average_item_size + flags_len; - nodes_updated - * KV::value_byte_cost_size_for_key_and_raw_value_lengths( - *average_key_size as u32, - average_value_len, - tree_type.inner_node_type(), - ) - } - EstimatedLayerSizes::Mix { - subtrees_size, - items_size, - references_size, - } => { - let total_weight = subtrees_size - .as_ref() - .map(|(_, _, _, weight)| *weight as u32) - .unwrap_or_default() - + items_size - .as_ref() - .map(|(_, _, _, weight)| *weight as u32) - .unwrap_or_default() - + references_size - .as_ref() - .map(|(_, _, _, weight)| *weight as u32) - .unwrap_or_default(); - if total_weight == 0 { - 0 - } else { - let weighted_nodes_updated = (nodes_updated as u64) - .checked_mul(total_weight as u64) - .ok_or(Error::Overflow("overflow for weights average cost"))?; - let tree_node_updates_cost = match subtrees_size { - None => 0, - Some((average_key_size, estimated_sum_trees, average_flags_size, weight)) => { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = LAYER_COST_SIZE + flags_len; - let sum_tree_addition = - estimated_sum_trees.estimated_size(grove_version)?; - let cost = KV::layered_value_byte_cost_size_for_key_and_value_lengths( - *average_key_size as u32, - value_len, - tree_type.inner_node_type(), - ) + sum_tree_addition; - (*weight as u64) - .checked_mul(cost as u64) - .ok_or(Error::Overflow("overflow for mixed tree nodes updates"))? - } - }; - let item_node_updates_cost = match items_size { - None => 0, - Some((average_key_size, average_value_size, average_flags_size, weight)) => { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = average_value_size + flags_len; - let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( - *average_key_size as u32, - value_len, - tree_type.inner_node_type(), - ); - (*weight as u64) - .checked_mul(cost as u64) - .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? - } - }; - let reference_node_updates_cost = match references_size { - None => 0, - Some((average_key_size, average_value_size, average_flags_size, weight)) => { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = average_value_size + flags_len; - let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( - *average_key_size as u32, - value_len, - tree_type.inner_node_type(), - ); - (*weight as u64) - .checked_mul(cost as u64) - .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? - } - }; - - let total_updates_cost = tree_node_updates_cost - .checked_add(item_node_updates_cost) - .and_then(|c| c.checked_add(reference_node_updates_cost)) - .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; - let total_replaced_bytes = total_updates_cost / weighted_nodes_updated; - if total_replaced_bytes > u32::MAX as u64 { - return Err(Error::Overflow( - "overflow for total replaced bytes more than u32 max", - )); - } - total_replaced_bytes as u32 - } - } - }; - cost.storage_loaded_bytes += match estimated_layer_sizes { - EstimatedLayerSizes::AllSubtrees( - average_key_size, - estimated_sum_trees, - average_flags_size, - ) => { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = LAYER_COST_SIZE + flags_len; - let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; - nodes_updated - * KV::layered_node_byte_cost_size_for_key_and_value_lengths( - *average_key_size as u32, - value_len + sum_tree_addition, - tree_type.inner_node_type(), - ) - } - EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) - | EstimatedLayerSizes::AllReference( - average_key_size, - average_item_size, - average_flags_size, - ) => { - let flags_len = average_flags_size.unwrap_or(0); - let average_value_len = average_item_size + flags_len; - nodes_updated - * KV::node_byte_cost_size_for_key_and_raw_value_lengths( - *average_key_size as u32, - average_value_len, - tree_type.inner_node_type(), - ) - } - EstimatedLayerSizes::Mix { - subtrees_size, - items_size, - references_size, - } => { - let total_weight = subtrees_size - .as_ref() - .map(|(_, _, _, weight)| *weight as u32) - .unwrap_or_default() - + items_size - .as_ref() - .map(|(_, _, _, weight)| *weight as u32) - .unwrap_or_default() - + references_size - .as_ref() - .map(|(_, _, _, weight)| *weight as u32) - .unwrap_or_default(); - if total_weight == 0 { - 0 - } else { - let weighted_nodes_updated = (nodes_updated as u64) - .checked_mul(total_weight as u64) - .ok_or(Error::Overflow("overflow for weights average cost"))?; - let tree_node_updates_cost = subtrees_size - .as_ref() - .map( - |(average_key_size, estimated_sum_trees, average_flags_size, weight)| { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = LAYER_COST_SIZE + flags_len; - let sum_tree_addition = - estimated_sum_trees.estimated_size(grove_version)?; - let cost = KV::layered_node_byte_cost_size_for_key_and_value_lengths( - *average_key_size as u32, - value_len + sum_tree_addition, - tree_type.inner_node_type(), - ); - (*weight as u64) - .checked_mul(cost as u64) - .ok_or(Error::Overflow("overflow for mixed tree nodes updates")) - }, - ) - .unwrap_or(Ok(0))?; - let item_node_updates_cost = items_size - .as_ref() - .map( - |(average_key_size, average_value_size, average_flags_size, weight)| { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = average_value_size + flags_len; - let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( - *average_key_size as u32, - value_len, - tree_type.inner_node_type(), - ); - (*weight as u64) - .checked_mul(cost as u64) - .ok_or(Error::Overflow("overflow for mixed item nodes updates")) - }, - ) - .unwrap_or(Ok(0))?; - let reference_node_updates_cost = references_size - .as_ref() - .map( - |(average_key_size, average_value_size, average_flags_size, weight)| { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = average_value_size + flags_len; - let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( - *average_key_size as u32, - value_len, - TreeType::NormalTree.inner_node_type(), - ); - (*weight as u64) - .checked_mul(cost as u64) - .ok_or(Error::Overflow("overflow for mixed item nodes updates")) - }, - ) - .unwrap_or(Ok(0))?; - - let total_updates_cost = tree_node_updates_cost - .checked_add(item_node_updates_cost) - .and_then(|c| c.checked_add(reference_node_updates_cost)) - .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; - let total_loaded_bytes = total_updates_cost / weighted_nodes_updated; - if total_loaded_bytes > u32::MAX as u64 { - return Err(Error::Overflow( - "overflow for total replaced bytes more than u32 max", - )); - } - total_loaded_bytes as u32 - } - } - } as u64; - Ok(()) -} - -#[cfg(feature = "minimal")] -/// Add average case cost for propagating a merk -fn add_average_case_merk_propagate_v0( - cost: &mut OperationCost, - input: &EstimatedLayerInformation, - grove_version: &GroveVersion, -) -> Result<(), Error> { - let mut nodes_updated = 0; - // Propagation requires to recompute and write hashes up to the root - let EstimatedLayerInformation { - tree_type, - estimated_layer_count, - estimated_layer_sizes, - } = input; - let levels = estimated_layer_count.estimate_levels(); - nodes_updated += levels; - - if levels > 1 { - // we can get about 1 rotation, if there are more than 2 levels - nodes_updated += 1; - } - cost.seek_count += nodes_updated; - - cost.hash_node_calls += nodes_updated * 2; - - cost.storage_cost.replaced_bytes += match estimated_layer_sizes { - EstimatedLayerSizes::AllSubtrees( - average_key_size, - estimated_sum_trees, - average_flags_size, - ) => { - // it is normal to have LAYER_COST_SIZE here, as we add estimated sum tree - // additions right after - let value_len = LAYER_COST_SIZE - + average_flags_size - .map_or(0, |flags_len| flags_len + flags_len.required_space() as u32); - // in order to simplify calculations we get the estimated size and remove the - // cost for the basic merk - let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; - nodes_updated - * (KV::layered_value_byte_cost_size_for_key_and_value_lengths( - *average_key_size as u32, - value_len, - tree_type.inner_node_type(), - ) + sum_tree_addition) - } - EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) - | EstimatedLayerSizes::AllReference( - average_key_size, - average_item_size, - average_flags_size, - ) => { - let flags_len = average_flags_size.unwrap_or(0); - let average_value_len = average_item_size + flags_len; - nodes_updated - * KV::value_byte_cost_size_for_key_and_raw_value_lengths( - *average_key_size as u32, - average_value_len, - tree_type.inner_node_type(), - ) - } - EstimatedLayerSizes::Mix { - subtrees_size, - items_size, - references_size, - } => { - let total_weight = subtrees_size - .as_ref() - .map(|(_, _, _, weight)| *weight as u32) - .unwrap_or_default() - + items_size - .as_ref() - .map(|(_, _, _, weight)| *weight as u32) - .unwrap_or_default() - + references_size - .as_ref() - .map(|(_, _, _, weight)| *weight as u32) - .unwrap_or_default(); - if total_weight == 0 { - 0 - } else { - let weighted_nodes_updated = (nodes_updated as u64) - .checked_mul(total_weight as u64) - .ok_or(Error::Overflow("overflow for weights average cost"))?; - let tree_node_updates_cost = match subtrees_size { - None => 0, - Some((average_key_size, estimated_sum_trees, average_flags_size, weight)) => { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = LAYER_COST_SIZE + flags_len; - let sum_tree_addition = - estimated_sum_trees.estimated_size(grove_version)?; - let cost = KV::layered_value_byte_cost_size_for_key_and_value_lengths( - *average_key_size as u32, - value_len, - tree_type.inner_node_type(), - ) + sum_tree_addition; - (*weight as u64) - .checked_mul(cost as u64) - .ok_or(Error::Overflow("overflow for mixed tree nodes updates"))? - } - }; - let item_node_updates_cost = match items_size { - None => 0, - Some((average_key_size, average_value_size, average_flags_size, weight)) => { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = average_value_size + flags_len; - let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( - *average_key_size as u32, - value_len, - tree_type.inner_node_type(), - ); - (*weight as u64) - .checked_mul(cost as u64) - .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? - } - }; - let reference_node_updates_cost = match references_size { - None => 0, - Some((average_key_size, average_value_size, average_flags_size, weight)) => { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = average_value_size + flags_len; - let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( - *average_key_size as u32, - value_len, - tree_type.inner_node_type(), - ); - (*weight as u64) - .checked_mul(cost as u64) - .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? - } - }; - - let total_updates_cost = tree_node_updates_cost - .checked_add(item_node_updates_cost) - .and_then(|c| c.checked_add(reference_node_updates_cost)) - .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; - let total_replaced_bytes = total_updates_cost / weighted_nodes_updated; - if total_replaced_bytes > u32::MAX as u64 { - return Err(Error::Overflow( - "overflow for total replaced bytes more than u32 max", - )); - } - total_replaced_bytes as u32 - } - } - }; - cost.storage_loaded_bytes += match estimated_layer_sizes { - EstimatedLayerSizes::AllSubtrees( - average_key_size, - estimated_sum_trees, - average_flags_size, - ) => { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = LAYER_COST_SIZE + flags_len; - let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; - nodes_updated - * KV::layered_node_byte_cost_size_for_key_and_value_lengths( - *average_key_size as u32, - value_len + sum_tree_addition, - tree_type.inner_node_type(), - ) - } - EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) - | EstimatedLayerSizes::AllReference( - average_key_size, - average_item_size, - average_flags_size, - ) => { - let flags_len = average_flags_size.unwrap_or(0); - let average_value_len = average_item_size + flags_len; - nodes_updated - * KV::node_byte_cost_size_for_key_and_raw_value_lengths( - *average_key_size as u32, - average_value_len, - tree_type.inner_node_type(), - ) - } - EstimatedLayerSizes::Mix { - subtrees_size, - items_size, - references_size, - } => { - let total_weight = subtrees_size - .as_ref() - .map(|(_, _, _, weight)| *weight as u32) - .unwrap_or_default() - + items_size - .as_ref() - .map(|(_, _, _, weight)| *weight as u32) - .unwrap_or_default() - + references_size - .as_ref() - .map(|(_, _, _, weight)| *weight as u32) - .unwrap_or_default(); - if total_weight == 0 { - 0 - } else { - let weighted_nodes_updated = (nodes_updated as u64) - .checked_mul(total_weight as u64) - .ok_or(Error::Overflow("overflow for weights average cost"))?; - let tree_node_updates_cost = subtrees_size - .as_ref() - .map( - |(average_key_size, estimated_sum_trees, average_flags_size, weight)| { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = LAYER_COST_SIZE + flags_len; - let sum_tree_addition = - estimated_sum_trees.estimated_size(grove_version)?; - let cost = KV::layered_node_byte_cost_size_for_key_and_value_lengths( - *average_key_size as u32, - value_len + sum_tree_addition, - tree_type.inner_node_type(), - ); - (*weight as u64) - .checked_mul(cost as u64) - .ok_or(Error::Overflow("overflow for mixed tree nodes updates")) - }, - ) - .unwrap_or(Ok(0))?; - let item_node_updates_cost = items_size - .as_ref() - .map( - |(average_key_size, average_value_size, average_flags_size, weight)| { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = average_value_size + flags_len; - let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( - *average_key_size as u32, - value_len, - tree_type.inner_node_type(), - ); - (*weight as u64) - .checked_mul(cost as u64) - .ok_or(Error::Overflow("overflow for mixed item nodes updates")) - }, - ) - .unwrap_or(Ok(0))?; - let reference_node_updates_cost = references_size - .as_ref() - .map( - |(average_key_size, average_value_size, average_flags_size, weight)| { - let flags_len = average_flags_size.unwrap_or(0); - let value_len = average_value_size + flags_len; - let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( - *average_key_size as u32, - value_len, - tree_type.inner_node_type(), // this was changed in v1 - ); - (*weight as u64) - .checked_mul(cost as u64) - .ok_or(Error::Overflow("overflow for mixed item nodes updates")) - }, - ) - .unwrap_or(Ok(0))?; - - let total_updates_cost = tree_node_updates_cost - .checked_add(item_node_updates_cost) - .and_then(|c| c.checked_add(reference_node_updates_cost)) - .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; - let total_loaded_bytes = total_updates_cost / weighted_nodes_updated; - if total_loaded_bytes > u32::MAX as u64 { - return Err(Error::Overflow( - "overflow for total replaced bytes more than u32 max", - )); - } - total_loaded_bytes as u32 - } - } - } as u64; - Ok(()) -} - -#[cfg(test)] -mod tests { - use grovedb_costs::OperationCost; - use grovedb_version::version::GroveVersion; - - use super::*; - - #[test] - fn test_estimated_sum_trees_divide_by_zero() { - let estimated = EstimatedSumTrees::SomeSumTrees { - sum_trees_weight: 0, - big_sum_trees_weight: 0, - count_trees_weight: 0, - count_sum_trees_weight: 0, - non_sum_trees_weight: 0, - }; - let err = estimated - .estimated_size(GroveVersion::latest()) - .unwrap_err(); - assert!(matches!(err, Error::DivideByZero("weights add up to 0"))); - } - - #[test] - fn test_estimated_sum_trees_v0_formula_path() { - let estimated = EstimatedSumTrees::SomeSumTrees { - sum_trees_weight: 1, - big_sum_trees_weight: 0, - count_trees_weight: 0, - count_sum_trees_weight: 0, - non_sum_trees_weight: 3, - }; - let size = estimated.estimated_size(GroveVersion::first()).unwrap(); - assert_eq!(size, 6); - } - - #[test] - fn test_layered_flags_size_errors() { - let layer = EstimatedLayerSizes::AllItems(4, 10, Some(1)); - let err = layer.layered_flags_size().unwrap_err(); - assert!(matches!( - err, - Error::WrongEstimatedCostsElementTypeForLevel( - "this layer does not have costs for trees" - ) - )); - } - - #[test] - fn test_subtree_with_feature_and_flags_size_mix_missing_subtrees() { - let layer = EstimatedLayerSizes::Mix { - subtrees_size: None, - items_size: Some((4, 10, Some(1), 1)), - references_size: None, - }; - let err = layer - .subtree_with_feature_and_flags_size(GroveVersion::latest()) - .unwrap_err(); - assert!(matches!( - err, - Error::WrongEstimatedCostsElementTypeForLevel( - "this layer is a mix but doesn't have subtrees" - ) - )); - } - - #[test] - fn test_value_with_feature_and_flags_size_mix_without_weights() { - let layer = EstimatedLayerSizes::Mix { - subtrees_size: None, - items_size: None, - references_size: None, - }; - let err = layer - .value_with_feature_and_flags_size(GroveVersion::latest()) - .unwrap_err(); - assert!(matches!( - err, - Error::WrongEstimatedCostsElementTypeForLevel( - "this layer is a mix and does not have items, refs or trees" - ) - )); - } - - #[test] - fn test_estimated_layer_count_helpers() { - assert!(EstimatedLayerCount::ApproximateElements(0).estimated_to_be_empty()); - assert_eq!( - EstimatedLayerCount::ApproximateElements(7).estimate_levels(), - 3 - ); - assert_eq!( - EstimatedLayerCount::PotentiallyAtMaxElements.estimate_levels(), - 32 - ); - assert_eq!( - EstimatedLayerCount::EstimatedLevel(5, false).estimate_levels(), - 5 - ); - } - - #[test] - fn test_add_average_case_merk_propagate_version_mismatch_error() { - let mut custom_version = GroveVersion::latest().clone(); - custom_version - .merk_versions - .average_case_costs - .add_average_case_merk_propagate = 99; - - let layer_info = EstimatedLayerInformation { - tree_type: TreeType::NormalTree, - estimated_layer_count: EstimatedLayerCount::EstimatedLevel(1, false), - estimated_layer_sizes: EstimatedLayerSizes::AllItems(5, 20, None), - }; - - let mut cost = OperationCost::default(); - let err = - add_average_case_merk_propagate(&mut cost, &layer_info, &custom_version).unwrap_err(); - assert!(matches!( - err, - Error::VersionError( - grovedb_version::error::GroveVersionError::UnknownVersionMismatch { .. } - ) - )); - } - - #[test] - fn test_add_average_case_merk_propagate_all_items_updates_cost() { - let layer_info = EstimatedLayerInformation { - tree_type: TreeType::NormalTree, - estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), - estimated_layer_sizes: EstimatedLayerSizes::AllItems(8, 32, Some(2)), - }; - - let mut cost = OperationCost::default(); - add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::latest()).unwrap(); - - assert!(cost.seek_count > 0); - assert!(cost.hash_node_calls > 0); - assert!(cost.storage_cost.replaced_bytes > 0); - assert!(cost.storage_loaded_bytes > 0); - } - - // ========================================================================= - // v0 propagation (covers add_average_case_merk_propagate_v0 entirely) - // ========================================================================= - - #[test] - fn test_propagate_v0_all_subtrees() { - let layer_info = EstimatedLayerInformation { - tree_type: TreeType::NormalTree, - estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), - estimated_layer_sizes: EstimatedLayerSizes::AllSubtrees( - 8, - EstimatedSumTrees::NoSumTrees, - Some(4), - ), - }; - let mut cost = OperationCost::default(); - add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::first()).unwrap(); - assert!(cost.seek_count > 0); - assert!(cost.storage_cost.replaced_bytes > 0); - assert!(cost.storage_loaded_bytes > 0); - } - - #[test] - fn test_propagate_v0_all_items() { - let layer_info = EstimatedLayerInformation { - tree_type: TreeType::NormalTree, - estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), - estimated_layer_sizes: EstimatedLayerSizes::AllItems(8, 32, Some(2)), - }; - let mut cost = OperationCost::default(); - add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::first()).unwrap(); - assert!(cost.seek_count > 0); - assert!(cost.storage_cost.replaced_bytes > 0); - assert!(cost.storage_loaded_bytes > 0); - } - - #[test] - fn test_propagate_v0_all_reference() { - let layer_info = EstimatedLayerInformation { - tree_type: TreeType::NormalTree, - estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), - estimated_layer_sizes: EstimatedLayerSizes::AllReference(8, 24, Some(2)), - }; - let mut cost = OperationCost::default(); - add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::first()).unwrap(); - assert!(cost.seek_count > 0); - assert!(cost.storage_cost.replaced_bytes > 0); - assert!(cost.storage_loaded_bytes > 0); - } - - #[test] - fn test_propagate_v0_mix() { - let layer_info = EstimatedLayerInformation { - tree_type: TreeType::NormalTree, - estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), - estimated_layer_sizes: EstimatedLayerSizes::Mix { - subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 2)), - items_size: Some((8, 32, Some(2), 3)), - references_size: Some((8, 24, Some(1), 1)), - }, - }; - let mut cost = OperationCost::default(); - add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::first()).unwrap(); - assert!(cost.seek_count > 0); - assert!(cost.storage_cost.replaced_bytes > 0); - assert!(cost.storage_loaded_bytes > 0); - } - - // ========================================================================= - // v1 propagation branches not yet covered (Mix, AllReference) - // ========================================================================= - - #[test] - fn test_propagate_v1_all_reference() { - let layer_info = EstimatedLayerInformation { - tree_type: TreeType::NormalTree, - estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), - estimated_layer_sizes: EstimatedLayerSizes::AllReference(8, 24, Some(2)), - }; - let mut cost = OperationCost::default(); - add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::latest()).unwrap(); - assert!(cost.seek_count > 0); - assert!(cost.storage_cost.replaced_bytes > 0); - assert!(cost.storage_loaded_bytes > 0); - } - - #[test] - fn test_propagate_v1_mix() { - let layer_info = EstimatedLayerInformation { - tree_type: TreeType::NormalTree, - estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), - estimated_layer_sizes: EstimatedLayerSizes::Mix { - subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 2)), - items_size: Some((8, 32, Some(2), 3)), - references_size: Some((8, 24, Some(1), 1)), - }, - }; - let mut cost = OperationCost::default(); - add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::latest()).unwrap(); - assert!(cost.seek_count > 0); - assert!(cost.storage_cost.replaced_bytes > 0); - assert!(cost.storage_loaded_bytes > 0); - } - - // ========================================================================= - // Utility functions with zero coverage - // ========================================================================= - - #[test] - fn test_add_average_case_merk_has_value() { - let mut cost = OperationCost::default(); - add_average_case_merk_has_value(&mut cost, 32, 100); - assert_eq!(cost.seek_count, 1); - assert_eq!(cost.storage_loaded_bytes, 132); - } - - #[test] - fn test_node_hash_update_count() { - let count = node_hash_update_count(); - // hash of 3 * HASH_LENGTH bytes: 1 + (bytes - 1) / HASH_BLOCK_SIZE - assert!(count > 0); - } - - #[test] - fn test_add_average_case_merk_root_hash() { - let mut cost = OperationCost::default(); - add_average_case_merk_root_hash(&mut cost); - assert!(cost.hash_node_calls > 0); - } - - // ========================================================================= - // EstimatedSumTrees variant arms not yet covered - // ========================================================================= - - #[test] - fn test_estimated_sum_trees_all_variant_arms() { - let v = GroveVersion::latest(); - let sum = EstimatedSumTrees::AllSumTrees.estimated_size(v).unwrap(); - assert!(sum > 0); - let big = EstimatedSumTrees::AllBigSumTrees.estimated_size(v).unwrap(); - assert!(big > 0); - let count = EstimatedSumTrees::AllCountTrees.estimated_size(v).unwrap(); - assert!(count > 0); - let count_sum = EstimatedSumTrees::AllCountSumTrees - .estimated_size(v) - .unwrap(); - assert!(count_sum > 0); - } - - // ========================================================================= - // EstimatedLayerCount edge case - // ========================================================================= - - #[test] - fn test_estimate_levels_u32_max() { - assert_eq!( - EstimatedLayerCount::ApproximateElements(u32::MAX).estimate_levels(), - 32 - ); - } - - // ========================================================================= - // EstimatedLayerSizes method branches - // ========================================================================= - - #[test] - fn test_layered_flags_size_mix_with_subtrees() { - let layer = EstimatedLayerSizes::Mix { - subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 1)), - items_size: Some((8, 32, Some(2), 1)), - references_size: None, - }; - let flags = layer.layered_flags_size().unwrap(); - assert_eq!(*flags, Some(4)); - } - - #[test] - fn test_layered_flags_size_mix_without_subtrees() { - let layer = EstimatedLayerSizes::Mix { - subtrees_size: None, - items_size: Some((8, 32, Some(2), 1)), - references_size: None, - }; - assert!(layer.layered_flags_size().is_err()); - } - - #[test] - fn test_subtree_size_all_subtrees() { - let layer = EstimatedLayerSizes::AllSubtrees(8, EstimatedSumTrees::NoSumTrees, Some(4)); - let size = layer - .subtree_with_feature_and_flags_size(GroveVersion::latest()) - .unwrap(); - assert_eq!(size, 7); // NoSumTrees=0, flags=4, base=3 - } - - #[test] - fn test_subtree_size_mix_with_subtrees() { - let layer = EstimatedLayerSizes::Mix { - subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 1)), - items_size: None, - references_size: None, - }; - let size = layer - .subtree_with_feature_and_flags_size(GroveVersion::latest()) - .unwrap(); - assert_eq!(size, 7); - } - - #[test] - fn test_subtree_size_non_subtree_errors() { - let layer = EstimatedLayerSizes::AllReference(8, 24, Some(2)); - assert!(layer - .subtree_with_feature_and_flags_size(GroveVersion::latest()) - .is_err()); - } - - #[test] - fn test_value_size_all_reference() { - let layer = EstimatedLayerSizes::AllReference(8, 24, Some(2)); - let size = layer - .value_with_feature_and_flags_size(GroveVersion::latest()) - .unwrap(); - assert_eq!(size, 24 + 2 + 5); // value + flags + 5 for refs - } - - #[test] - fn test_value_size_all_subtrees() { - let layer = EstimatedLayerSizes::AllSubtrees(8, EstimatedSumTrees::NoSumTrees, Some(4)); - let size = layer - .value_with_feature_and_flags_size(GroveVersion::latest()) - .unwrap(); - assert_eq!(size, 7); - } - - #[test] - fn test_value_size_mix_weighted_combination() { - let layer = EstimatedLayerSizes::Mix { - subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 2)), - items_size: Some((8, 32, Some(2), 3)), - references_size: Some((8, 24, Some(1), 1)), - }; - let size = layer - .value_with_feature_and_flags_size(GroveVersion::latest()) - .unwrap(); - // item_size=32+2+3=37, ref_size=24+1+5=30, subtree_size=0+4+3=7 - // combined_weight=3+1+2=6, result=(37+30+7)/6=12 - assert_eq!(size, 12); - } - - #[test] - fn test_value_size_mix_only_subtrees() { - let layer = EstimatedLayerSizes::Mix { - subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 2)), - items_size: None, - references_size: None, - }; - let size = layer - .value_with_feature_and_flags_size(GroveVersion::latest()) - .unwrap(); - assert_eq!(size, 7); - } - - #[test] - fn test_value_size_mix_only_refs() { - let layer = EstimatedLayerSizes::Mix { - subtrees_size: None, - items_size: None, - references_size: Some((8, 24, Some(1), 1)), - }; - let size = layer - .value_with_feature_and_flags_size(GroveVersion::latest()) - .unwrap(); - assert_eq!(size, 24 + 1 + 5); - } - - #[test] - fn test_value_size_mix_only_items() { - let layer = EstimatedLayerSizes::Mix { - subtrees_size: None, - items_size: Some((8, 32, Some(2), 3)), - references_size: None, - }; - let size = layer - .value_with_feature_and_flags_size(GroveVersion::latest()) - .unwrap(); - assert_eq!(size, 32 + 2 + 3); - } -} diff --git a/merk/src/estimated_costs/average_case_costs/add_average_case_merk_propagate/mod.rs b/merk/src/estimated_costs/average_case_costs/add_average_case_merk_propagate/mod.rs new file mode 100644 index 000000000..60a7ea7eb --- /dev/null +++ b/merk/src/estimated_costs/average_case_costs/add_average_case_merk_propagate/mod.rs @@ -0,0 +1,53 @@ +//! Versioned dispatch for [`add_average_case_merk_propagate`] (and the +//! cost-wrapping convenience helper [`average_case_merk_propagate`]). +//! +//! Each `v*.rs` defines one `add_average_case_merk_propagate_v*` free +//! function. The dispatcher below selects the implementation by +//! `GroveVersion::merk_versions.average_case_costs.add_average_case_merk_propagate`. + +mod v0; +mod v1; +mod v2; + +use v0::add_average_case_merk_propagate_v0; +use v1::add_average_case_merk_propagate_v1; +use v2::add_average_case_merk_propagate_v2; + +use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; + +use super::EstimatedLayerInformation; +use crate::error::Error; + +/// Average case cost of propagating a merk +pub fn average_case_merk_propagate( + input: &EstimatedLayerInformation, + grove_version: &GroveVersion, +) -> CostResult<(), Error> { + let mut cost = OperationCost::default(); + add_average_case_merk_propagate(&mut cost, input, grove_version).wrap_with_cost(cost) +} + +/// Add average case cost for propagating a merk +pub fn add_average_case_merk_propagate( + cost: &mut OperationCost, + input: &EstimatedLayerInformation, + grove_version: &GroveVersion, +) -> Result<(), Error> { + match grove_version + .merk_versions + .average_case_costs + .add_average_case_merk_propagate + { + 0 => add_average_case_merk_propagate_v0(cost, input, grove_version), + 1 => add_average_case_merk_propagate_v1(cost, input, grove_version), + 2 => add_average_case_merk_propagate_v2(cost, input, grove_version), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "add_average_case_merk_propagate".to_string(), + known_versions: vec![0, 1, 2], + received: version, + }, + )), + } +} diff --git a/merk/src/estimated_costs/average_case_costs/add_average_case_merk_propagate/v0.rs b/merk/src/estimated_costs/average_case_costs/add_average_case_merk_propagate/v0.rs new file mode 100644 index 000000000..f542a1f30 --- /dev/null +++ b/merk/src/estimated_costs/average_case_costs/add_average_case_merk_propagate/v0.rs @@ -0,0 +1,424 @@ +//! `add_average_case_merk_propagate` v0 — grove v1 dispatch path +//! (historical baseline). Inherits the pre-existing buggy Mix divisor; +//! see `add_average_case_merk_propagate_v2` for the corrected formula. +//! +//! Uses `tree_type.inner_node_type()` for the Mix reference-cost +//! arms throughout (v1 changes the `storage_loaded_bytes` arm to +//! `TreeType::NormalTree.inner_node_type()`; the inline `// this was +//! changed in v1` comment marks that line). + +use grovedb_costs::OperationCost; +use grovedb_version::version::GroveVersion; +use integer_encoding::VarInt; + +use crate::{ + error::Error, + estimated_costs::{ + average_case_costs::{EstimatedLayerInformation, EstimatedLayerSizes}, + LAYER_COST_SIZE, + }, + tree::kv::KV, +}; + +pub(super) fn add_average_case_merk_propagate_v0( + cost: &mut OperationCost, + input: &EstimatedLayerInformation, + grove_version: &GroveVersion, +) -> Result<(), Error> { + let mut nodes_updated = 0; + // Propagation requires to recompute and write hashes up to the root + let EstimatedLayerInformation { + tree_type, + estimated_layer_count, + estimated_layer_sizes, + } = input; + let levels = estimated_layer_count.estimate_levels(); + nodes_updated += levels; + + if levels > 1 { + // we can get about 1 rotation, if there are more than 2 levels + nodes_updated += 1; + } + cost.seek_count += nodes_updated; + + cost.hash_node_calls += nodes_updated * 2; + + cost.storage_cost.replaced_bytes += match estimated_layer_sizes { + EstimatedLayerSizes::AllSubtrees( + average_key_size, + estimated_sum_trees, + average_flags_size, + ) => { + // it is normal to have LAYER_COST_SIZE here, as we add estimated sum tree + // additions right after + let value_len = LAYER_COST_SIZE + + average_flags_size + .map_or(0, |flags_len| flags_len + flags_len.required_space() as u32); + // in order to simplify calculations we get the estimated size and remove the + // cost for the basic merk + let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; + nodes_updated + * (KV::layered_value_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ) + sum_tree_addition) + } + EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) + | EstimatedLayerSizes::AllReference( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len; + nodes_updated + * KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::AllItemsWithSumItem( + average_key_size, + average_item_size, + average_flags_size, + ) + | EstimatedLayerSizes::AllReferencesWithSumItem( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len + 10; + nodes_updated + * KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::Mix { + subtrees_size, + items_size, + references_size, + items_with_sum_item_size, + references_with_sum_item_size, + } => { + let total_weight = subtrees_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_with_sum_item_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_with_sum_item_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default(); + if total_weight == 0 { + 0 + } else { + let weighted_nodes_updated = (nodes_updated as u64) + .checked_mul(total_weight as u64) + .ok_or(Error::Overflow("overflow for weights average cost"))?; + let tree_node_updates_cost = match subtrees_size { + None => 0, + Some((average_key_size, estimated_sum_trees, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = + estimated_sum_trees.estimated_size(grove_version)?; + let cost = KV::layered_value_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ) + sum_tree_addition; + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed tree nodes updates"))? + } + }; + let item_node_updates_cost = match items_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? + } + }; + let reference_node_updates_cost = match references_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? + } + }; + let item_with_sum_node_updates_cost = match items_with_sum_item_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len + 10; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow( + "overflow for mixed item-with-sum nodes updates", + ))? + } + }; + let reference_with_sum_node_updates_cost = match references_with_sum_item_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len + 10; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow( + "overflow for mixed reference-with-sum nodes updates", + ))? + } + }; + + let total_updates_cost = tree_node_updates_cost + .checked_add(item_node_updates_cost) + .and_then(|c| c.checked_add(reference_node_updates_cost)) + .and_then(|c| c.checked_add(item_with_sum_node_updates_cost)) + .and_then(|c| c.checked_add(reference_with_sum_node_updates_cost)) + .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; + let total_replaced_bytes = total_updates_cost / weighted_nodes_updated; + if total_replaced_bytes > u32::MAX as u64 { + return Err(Error::Overflow( + "overflow for total replaced bytes more than u32 max", + )); + } + total_replaced_bytes as u32 + } + } + }; + cost.storage_loaded_bytes += match estimated_layer_sizes { + EstimatedLayerSizes::AllSubtrees( + average_key_size, + estimated_sum_trees, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; + nodes_updated + * KV::layered_node_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len + sum_tree_addition, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) + | EstimatedLayerSizes::AllReference( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len; + nodes_updated + * KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::AllItemsWithSumItem( + average_key_size, + average_item_size, + average_flags_size, + ) + | EstimatedLayerSizes::AllReferencesWithSumItem( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len + 10; + nodes_updated + * KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::Mix { + subtrees_size, + items_size, + references_size, + items_with_sum_item_size, + references_with_sum_item_size, + } => { + let total_weight = subtrees_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_with_sum_item_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_with_sum_item_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default(); + if total_weight == 0 { + 0 + } else { + let weighted_nodes_updated = (nodes_updated as u64) + .checked_mul(total_weight as u64) + .ok_or(Error::Overflow("overflow for weights average cost"))?; + let tree_node_updates_cost = subtrees_size + .as_ref() + .map( + |(average_key_size, estimated_sum_trees, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = + estimated_sum_trees.estimated_size(grove_version)?; + let cost = KV::layered_node_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len + sum_tree_addition, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed tree nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + let item_node_updates_cost = items_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + let reference_node_updates_cost = references_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), // this was changed in v1 + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + let item_with_sum_node_updates_cost = items_with_sum_item_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len + 10; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow( + "overflow for mixed item-with-sum nodes updates", + )) + }, + ) + .unwrap_or(Ok(0))?; + let reference_with_sum_node_updates_cost = references_with_sum_item_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len + 10; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow( + "overflow for mixed reference-with-sum nodes updates", + )) + }, + ) + .unwrap_or(Ok(0))?; + + let total_updates_cost = tree_node_updates_cost + .checked_add(item_node_updates_cost) + .and_then(|c| c.checked_add(reference_node_updates_cost)) + .and_then(|c| c.checked_add(item_with_sum_node_updates_cost)) + .and_then(|c| c.checked_add(reference_with_sum_node_updates_cost)) + .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; + let total_loaded_bytes = total_updates_cost / weighted_nodes_updated; + if total_loaded_bytes > u32::MAX as u64 { + return Err(Error::Overflow( + "overflow for total replaced bytes more than u32 max", + )); + } + total_loaded_bytes as u32 + } + } + } as u64; + Ok(()) +} diff --git a/merk/src/estimated_costs/average_case_costs/add_average_case_merk_propagate/v1.rs b/merk/src/estimated_costs/average_case_costs/add_average_case_merk_propagate/v1.rs new file mode 100644 index 000000000..7cb5972a2 --- /dev/null +++ b/merk/src/estimated_costs/average_case_costs/add_average_case_merk_propagate/v1.rs @@ -0,0 +1,429 @@ +//! `add_average_case_merk_propagate` v1 — grove v2 dispatch path. +//! +//! Identical to v0 except the Mix `storage_loaded_bytes` arm's +//! reference-cost lookup changed from `tree_type.inner_node_type()` to +//! `TreeType::NormalTree.inner_node_type()`. +//! +//! Inherits v0's pre-existing Mix divisor bug +//! (`total_updates_cost / (nodes_updated · total_weight)`); see v2 for +//! the corrected formula. + +use grovedb_costs::OperationCost; +use grovedb_version::version::GroveVersion; +use integer_encoding::VarInt; + +use crate::{ + error::Error, + estimated_costs::{ + average_case_costs::{EstimatedLayerInformation, EstimatedLayerSizes}, + LAYER_COST_SIZE, + }, + tree::kv::KV, + tree_type::TreeType, +}; + +pub(super) fn add_average_case_merk_propagate_v1( + cost: &mut OperationCost, + input: &EstimatedLayerInformation, + grove_version: &GroveVersion, +) -> Result<(), Error> { + let mut nodes_updated = 0; + // Propagation requires to recompute and write hashes up to the root + let EstimatedLayerInformation { + tree_type, + estimated_layer_count, + estimated_layer_sizes, + } = input; + let levels = estimated_layer_count.estimate_levels(); + nodes_updated += levels; + + if levels > 1 { + // we can get about 1 rotation, if there are more than 2 levels + nodes_updated += 1; + } + cost.seek_count += nodes_updated; + + cost.hash_node_calls += nodes_updated * 2; + + cost.storage_cost.replaced_bytes += match estimated_layer_sizes { + EstimatedLayerSizes::AllSubtrees( + average_key_size, + estimated_sum_trees, + average_flags_size, + ) => { + // it is normal to have LAYER_COST_SIZE here, as we add estimated sum tree + // additions right after + let value_len = LAYER_COST_SIZE + + average_flags_size + .map_or(0, |flags_len| flags_len + flags_len.required_space() as u32); + // in order to simplify calculations we get the estimated size and remove the + // cost for the basic merk + let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; + nodes_updated + * (KV::layered_value_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ) + sum_tree_addition) + } + EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) + | EstimatedLayerSizes::AllReference( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len; + nodes_updated + * KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::AllItemsWithSumItem( + average_key_size, + average_item_size, + average_flags_size, + ) + | EstimatedLayerSizes::AllReferencesWithSumItem( + average_key_size, + average_item_size, + average_flags_size, + ) => { + // +10 for the worst-case i64 sum_value varint, matching + // `required_item_with_sum_item_space` / + // `required_reference_with_sum_item_space`. + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len + 10; + nodes_updated + * KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::Mix { + subtrees_size, + items_size, + references_size, + items_with_sum_item_size, + references_with_sum_item_size, + } => { + let total_weight = subtrees_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_with_sum_item_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_with_sum_item_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default(); + if total_weight == 0 { + 0 + } else { + let weighted_nodes_updated = (nodes_updated as u64) + .checked_mul(total_weight as u64) + .ok_or(Error::Overflow("overflow for weights average cost"))?; + let tree_node_updates_cost = match subtrees_size { + None => 0, + Some((average_key_size, estimated_sum_trees, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = + estimated_sum_trees.estimated_size(grove_version)?; + let cost = KV::layered_value_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ) + sum_tree_addition; + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed tree nodes updates"))? + } + }; + let item_node_updates_cost = match items_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? + } + }; + let reference_node_updates_cost = match references_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? + } + }; + let item_with_sum_node_updates_cost = match items_with_sum_item_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len + 10; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow( + "overflow for mixed item-with-sum nodes updates", + ))? + } + }; + let reference_with_sum_node_updates_cost = match references_with_sum_item_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len + 10; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow( + "overflow for mixed reference-with-sum nodes updates", + ))? + } + }; + + let total_updates_cost = tree_node_updates_cost + .checked_add(item_node_updates_cost) + .and_then(|c| c.checked_add(reference_node_updates_cost)) + .and_then(|c| c.checked_add(item_with_sum_node_updates_cost)) + .and_then(|c| c.checked_add(reference_with_sum_node_updates_cost)) + .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; + let total_replaced_bytes = total_updates_cost / weighted_nodes_updated; + if total_replaced_bytes > u32::MAX as u64 { + return Err(Error::Overflow( + "overflow for total replaced bytes more than u32 max", + )); + } + total_replaced_bytes as u32 + } + } + }; + cost.storage_loaded_bytes += match estimated_layer_sizes { + EstimatedLayerSizes::AllSubtrees( + average_key_size, + estimated_sum_trees, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; + nodes_updated + * KV::layered_node_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len + sum_tree_addition, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) + | EstimatedLayerSizes::AllReference( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len; + nodes_updated + * KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::AllItemsWithSumItem( + average_key_size, + average_item_size, + average_flags_size, + ) + | EstimatedLayerSizes::AllReferencesWithSumItem( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len + 10; + nodes_updated + * KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::Mix { + subtrees_size, + items_size, + references_size, + items_with_sum_item_size, + references_with_sum_item_size, + } => { + let total_weight = subtrees_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_with_sum_item_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_with_sum_item_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default(); + if total_weight == 0 { + 0 + } else { + let weighted_nodes_updated = (nodes_updated as u64) + .checked_mul(total_weight as u64) + .ok_or(Error::Overflow("overflow for weights average cost"))?; + let tree_node_updates_cost = subtrees_size + .as_ref() + .map( + |(average_key_size, estimated_sum_trees, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = + estimated_sum_trees.estimated_size(grove_version)?; + let cost = KV::layered_node_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len + sum_tree_addition, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed tree nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + let item_node_updates_cost = items_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + let reference_node_updates_cost = references_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + TreeType::NormalTree.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + let item_with_sum_node_updates_cost = items_with_sum_item_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len + 10; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow( + "overflow for mixed item-with-sum nodes updates", + )) + }, + ) + .unwrap_or(Ok(0))?; + let reference_with_sum_node_updates_cost = references_with_sum_item_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len + 10; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + TreeType::NormalTree.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow( + "overflow for mixed reference-with-sum nodes updates", + )) + }, + ) + .unwrap_or(Ok(0))?; + + let total_updates_cost = tree_node_updates_cost + .checked_add(item_node_updates_cost) + .and_then(|c| c.checked_add(reference_node_updates_cost)) + .and_then(|c| c.checked_add(item_with_sum_node_updates_cost)) + .and_then(|c| c.checked_add(reference_with_sum_node_updates_cost)) + .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; + let total_loaded_bytes = total_updates_cost / weighted_nodes_updated; + if total_loaded_bytes > u32::MAX as u64 { + return Err(Error::Overflow( + "overflow for total replaced bytes more than u32 max", + )); + } + total_loaded_bytes as u32 + } + } + } as u64; + Ok(()) +} diff --git a/merk/src/estimated_costs/average_case_costs/add_average_case_merk_propagate/v2.rs b/merk/src/estimated_costs/average_case_costs/add_average_case_merk_propagate/v2.rs new file mode 100644 index 000000000..a36decb81 --- /dev/null +++ b/merk/src/estimated_costs/average_case_costs/add_average_case_merk_propagate/v2.rs @@ -0,0 +1,430 @@ +//! `add_average_case_merk_propagate` v2 — grove v3 dispatch path. +//! +//! Fixes the pre-existing Mix divisor bug. v1 computed +//! `total_updates_cost / (nodes_updated · total_weight)`, which gave +//! the per-node weighted-average cost divided by `nodes_updated` +//! instead of the per-layer weighted total — equivalent to +//! `cost / nodes_updated²` for an items-only Mix vs the same +//! `AllItems` layer. +//! +//! v2 computes `nodes_updated · total_updates_cost / total_weight`, +//! matching what the `AllItems` / `AllSubtrees` arms already do +//! (`nodes_updated · cost_per_node`). +//! +//! Uses `tree_type.inner_node_type()` for the Mix reference-cost arms +//! throughout (same as v0; v1's `storage_loaded_bytes` arm used +//! `TreeType::NormalTree.inner_node_type()` — see v1's doc comment). + +use grovedb_costs::OperationCost; +use grovedb_version::version::GroveVersion; +use integer_encoding::VarInt; + +use crate::{ + error::Error, + estimated_costs::{ + average_case_costs::{EstimatedLayerInformation, EstimatedLayerSizes}, + LAYER_COST_SIZE, + }, + tree::kv::KV, +}; + +pub(super) fn add_average_case_merk_propagate_v2( + cost: &mut OperationCost, + input: &EstimatedLayerInformation, + grove_version: &GroveVersion, +) -> Result<(), Error> { + let mut nodes_updated = 0; + let EstimatedLayerInformation { + tree_type, + estimated_layer_count, + estimated_layer_sizes, + } = input; + let levels = estimated_layer_count.estimate_levels(); + nodes_updated += levels; + + if levels > 1 { + nodes_updated += 1; + } + cost.seek_count += nodes_updated; + + cost.hash_node_calls += nodes_updated * 2; + + cost.storage_cost.replaced_bytes += match estimated_layer_sizes { + EstimatedLayerSizes::AllSubtrees( + average_key_size, + estimated_sum_trees, + average_flags_size, + ) => { + let value_len = LAYER_COST_SIZE + + average_flags_size + .map_or(0, |flags_len| flags_len + flags_len.required_space() as u32); + let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; + nodes_updated + * (KV::layered_value_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ) + sum_tree_addition) + } + EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) + | EstimatedLayerSizes::AllReference( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len; + nodes_updated + * KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::AllItemsWithSumItem( + average_key_size, + average_item_size, + average_flags_size, + ) + | EstimatedLayerSizes::AllReferencesWithSumItem( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len + 10; + nodes_updated + * KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::Mix { + subtrees_size, + items_size, + references_size, + items_with_sum_item_size, + references_with_sum_item_size, + } => { + let total_weight = subtrees_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_with_sum_item_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_with_sum_item_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default(); + if total_weight == 0 { + 0 + } else { + let tree_node_updates_cost = match subtrees_size { + None => 0, + Some((average_key_size, estimated_sum_trees, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = + estimated_sum_trees.estimated_size(grove_version)?; + let cost = KV::layered_value_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ) + sum_tree_addition; + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed tree nodes updates"))? + } + }; + let item_node_updates_cost = match items_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? + } + }; + let reference_node_updates_cost = match references_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? + } + }; + let item_with_sum_node_updates_cost = match items_with_sum_item_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len + 10; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow( + "overflow for mixed item-with-sum nodes updates", + ))? + } + }; + let reference_with_sum_node_updates_cost = match references_with_sum_item_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len + 10; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow( + "overflow for mixed reference-with-sum nodes updates", + ))? + } + }; + + let total_updates_cost = tree_node_updates_cost + .checked_add(item_node_updates_cost) + .and_then(|c| c.checked_add(reference_node_updates_cost)) + .and_then(|c| c.checked_add(item_with_sum_node_updates_cost)) + .and_then(|c| c.checked_add(reference_with_sum_node_updates_cost)) + .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; + // v2 fix: scale by nodes_updated, divide by total_weight + // (the per-node weighted average × nodes_updated). + // Multiply first to preserve precision in integer + // arithmetic. + let scaled = (nodes_updated as u64) + .checked_mul(total_updates_cost) + .ok_or(Error::Overflow("overflow scaling mixed propagate cost"))?; + let total_replaced_bytes = scaled / total_weight as u64; + if total_replaced_bytes > u32::MAX as u64 { + return Err(Error::Overflow( + "overflow for total replaced bytes more than u32 max", + )); + } + total_replaced_bytes as u32 + } + } + }; + cost.storage_loaded_bytes += match estimated_layer_sizes { + EstimatedLayerSizes::AllSubtrees( + average_key_size, + estimated_sum_trees, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; + nodes_updated + * KV::layered_node_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len + sum_tree_addition, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) + | EstimatedLayerSizes::AllReference( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len; + nodes_updated + * KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::AllItemsWithSumItem( + average_key_size, + average_item_size, + average_flags_size, + ) + | EstimatedLayerSizes::AllReferencesWithSumItem( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len + 10; + nodes_updated + * KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::Mix { + subtrees_size, + items_size, + references_size, + items_with_sum_item_size, + references_with_sum_item_size, + } => { + let total_weight = subtrees_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_with_sum_item_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_with_sum_item_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default(); + if total_weight == 0 { + 0 + } else { + let tree_node_updates_cost = subtrees_size + .as_ref() + .map( + |(average_key_size, estimated_sum_trees, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = + estimated_sum_trees.estimated_size(grove_version)?; + let cost = KV::layered_node_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len + sum_tree_addition, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed tree nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + let item_node_updates_cost = items_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + let reference_node_updates_cost = references_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + let item_with_sum_node_updates_cost = items_with_sum_item_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len + 10; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow( + "overflow for mixed item-with-sum nodes updates", + )) + }, + ) + .unwrap_or(Ok(0))?; + let reference_with_sum_node_updates_cost = references_with_sum_item_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len + 10; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow( + "overflow for mixed reference-with-sum nodes updates", + )) + }, + ) + .unwrap_or(Ok(0))?; + + let total_updates_cost = tree_node_updates_cost + .checked_add(item_node_updates_cost) + .and_then(|c| c.checked_add(reference_node_updates_cost)) + .and_then(|c| c.checked_add(item_with_sum_node_updates_cost)) + .and_then(|c| c.checked_add(reference_with_sum_node_updates_cost)) + .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; + let scaled = (nodes_updated as u64) + .checked_mul(total_updates_cost) + .ok_or(Error::Overflow("overflow scaling mixed propagate cost"))?; + let total_loaded_bytes = scaled / total_weight as u64; + if total_loaded_bytes > u32::MAX as u64 { + return Err(Error::Overflow( + "overflow for total replaced bytes more than u32 max", + )); + } + total_loaded_bytes as u32 + } + } + } as u64; + Ok(()) +} diff --git a/merk/src/estimated_costs/average_case_costs/estimated_sum_trees_size/mod.rs b/merk/src/estimated_costs/average_case_costs/estimated_sum_trees_size/mod.rs new file mode 100644 index 000000000..ce0c976ac --- /dev/null +++ b/merk/src/estimated_costs/average_case_costs/estimated_sum_trees_size/mod.rs @@ -0,0 +1,46 @@ +//! Versioned dispatch for [`EstimatedSumTrees::estimated_size`]. +//! +//! Each `v*.rs` adds one `estimated_size_v*` method via a separate +//! `impl EstimatedSumTrees` block. The dispatcher below selects the +//! implementation by +//! `GroveVersion::merk_versions.average_case_costs.sum_tree_estimated_size`. + +mod v0; +mod v1; +mod v2; + +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; + +use super::EstimatedSumTrees; +use crate::error::Error; + +impl EstimatedSumTrees { + /// Returns the average per-node cost contribution of the sum/count + /// aggregate state for a tree with this distribution of leaf tree + /// types. Version-dispatched: + /// - v0: shipped formula from grove v1 (uses only sum/non-sum weights) + /// - v1: weighted by per-tree-type `inner_node_type().cost()` across + /// the four legacy aggregate weights (grove v2) + /// - v2: extends v1 with the four `provable_*` weights (grove v3) + pub(in crate::estimated_costs) fn estimated_size( + &self, + grove_version: &GroveVersion, + ) -> Result { + match grove_version + .merk_versions + .average_case_costs + .sum_tree_estimated_size + { + 0 => self.estimated_size_v0(), + 1 => self.estimated_size_v1(), + 2 => self.estimated_size_v2(), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "EstimatedSumTrees::estimated_size".to_string(), + known_versions: vec![0, 1, 2], + received: version, + }, + )), + } + } +} diff --git a/merk/src/estimated_costs/average_case_costs/estimated_sum_trees_size/v0.rs b/merk/src/estimated_costs/average_case_costs/estimated_sum_trees_size/v0.rs new file mode 100644 index 000000000..2a80efa56 --- /dev/null +++ b/merk/src/estimated_costs/average_case_costs/estimated_sum_trees_size/v0.rs @@ -0,0 +1,51 @@ +//! `EstimatedSumTrees::estimated_size` v0 — original grove v1 formula. +//! +//! v0 only consults `sum_trees_weight` and `non_sum_trees_weight`; the +//! other weight fields (`big_sum`, `count*`, `provable_*`) didn't exist +//! when this formula shipped and are ignored. The divisor is +//! `sum_trees + non_sum_trees`, guarded against division by zero — a +//! layer with only `big_sum_trees_weight` (or any count weight) set +//! would have a nonzero legacy total but a zero v0 denominator. + +use crate::{ + error::Error, estimated_costs::average_case_costs::EstimatedSumTrees, tree_type::TreeType, +}; + +impl EstimatedSumTrees { + pub(super) fn estimated_size_v0(&self) -> Result { + match self { + EstimatedSumTrees::NoSumTrees => Ok(0), + EstimatedSumTrees::SomeSumTrees { + sum_trees_weight, + non_sum_trees_weight, + .. + } => { + let denominator = *sum_trees_weight as u32 + *non_sum_trees_weight as u32; + if denominator == 0 { + return Err(Error::DivideByZero("weights add up to 0")); + } + Ok((*non_sum_trees_weight as u32 * 9) / denominator) + } + EstimatedSumTrees::AllSumTrees => Ok(TreeType::SumTree.inner_node_type().cost()), + EstimatedSumTrees::AllBigSumTrees => Ok(TreeType::BigSumTree.inner_node_type().cost()), + EstimatedSumTrees::AllCountTrees => Ok(TreeType::CountTree.inner_node_type().cost()), + EstimatedSumTrees::AllCountSumTrees => { + Ok(TreeType::CountSumTree.inner_node_type().cost()) + } + EstimatedSumTrees::AllProvableSumTrees => { + Ok(TreeType::ProvableSumTree.inner_node_type().cost()) + } + EstimatedSumTrees::AllProvableCountTrees => { + Ok(TreeType::ProvableCountTree.inner_node_type().cost()) + } + EstimatedSumTrees::AllProvableCountSumTrees => { + Ok(TreeType::ProvableCountSumTree.inner_node_type().cost()) + } + EstimatedSumTrees::AllProvableCountProvableSumTrees => { + Ok(TreeType::ProvableCountProvableSumTree + .inner_node_type() + .cost()) + } + } + } +} diff --git a/merk/src/estimated_costs/average_case_costs/estimated_sum_trees_size/v1.rs b/merk/src/estimated_costs/average_case_costs/estimated_sum_trees_size/v1.rs new file mode 100644 index 000000000..76114bfaa --- /dev/null +++ b/merk/src/estimated_costs/average_case_costs/estimated_sum_trees_size/v1.rs @@ -0,0 +1,77 @@ +//! `EstimatedSumTrees::estimated_size` v1 — grove v2 formula. +//! +//! Weighted average across the four legacy aggregate tree types +//! (`SumTree`, `BigSumTree`, `CountTree`, `CountSumTree`), using each +//! type's `inner_node_type().cost()` as the per-element contribution. +//! `non_sum_trees_weight` enters only via the divisor (it contributes 0 +//! to the numerator since `NormalNode.cost() == 0`). The four +//! `provable_*` weight fields didn't exist when this formula shipped +//! and are ignored. + +use crate::{ + error::Error, estimated_costs::average_case_costs::EstimatedSumTrees, tree_type::TreeType, +}; + +impl EstimatedSumTrees { + pub(super) fn estimated_size_v1(&self) -> Result { + match self { + EstimatedSumTrees::NoSumTrees => Ok(0), + EstimatedSumTrees::SomeSumTrees { + sum_trees_weight, + big_sum_trees_weight, + count_trees_weight, + count_sum_trees_weight, + non_sum_trees_weight, + .. + } => { + let total_weight = *sum_trees_weight as u32 + + *big_sum_trees_weight as u32 + + *count_trees_weight as u32 + + *count_sum_trees_weight as u32 + + *non_sum_trees_weight as u32; + if total_weight == 0 { + return Err(Error::DivideByZero("weights add up to 0")); + } + let estimated_size = (*sum_trees_weight as u32 + * TreeType::SumTree.inner_node_type().cost()) + .checked_add( + *big_sum_trees_weight as u32 * TreeType::BigSumTree.inner_node_type().cost(), + ) + .and_then(|sum| { + sum.checked_add( + *count_trees_weight as u32 * TreeType::CountTree.inner_node_type().cost(), + ) + }) + .and_then(|sum| { + sum.checked_add( + *count_sum_trees_weight as u32 + * TreeType::CountSumTree.inner_node_type().cost(), + ) + }) + .ok_or(Error::Overflow("Estimated size calculation overflowed"))?; + + Ok(estimated_size / total_weight) + } + EstimatedSumTrees::AllSumTrees => Ok(TreeType::SumTree.inner_node_type().cost()), + EstimatedSumTrees::AllBigSumTrees => Ok(TreeType::BigSumTree.inner_node_type().cost()), + EstimatedSumTrees::AllCountTrees => Ok(TreeType::CountTree.inner_node_type().cost()), + EstimatedSumTrees::AllCountSumTrees => { + Ok(TreeType::CountSumTree.inner_node_type().cost()) + } + EstimatedSumTrees::AllProvableSumTrees => { + Ok(TreeType::ProvableSumTree.inner_node_type().cost()) + } + EstimatedSumTrees::AllProvableCountTrees => { + Ok(TreeType::ProvableCountTree.inner_node_type().cost()) + } + EstimatedSumTrees::AllProvableCountSumTrees => { + Ok(TreeType::ProvableCountSumTree.inner_node_type().cost()) + } + EstimatedSumTrees::AllProvableCountProvableSumTrees => { + Ok(TreeType::ProvableCountProvableSumTree + .inner_node_type() + .cost()) + } + } + } +} diff --git a/merk/src/estimated_costs/average_case_costs/estimated_sum_trees_size/v2.rs b/merk/src/estimated_costs/average_case_costs/estimated_sum_trees_size/v2.rs new file mode 100644 index 000000000..41790139d --- /dev/null +++ b/merk/src/estimated_costs/average_case_costs/estimated_sum_trees_size/v2.rs @@ -0,0 +1,111 @@ +//! `EstimatedSumTrees::estimated_size` v2 — grove v3 formula. +//! +//! Extends v1 with the four `provable_*` weight fields covering +//! `ProvableSumTree`, `ProvableCountTree`, `ProvableCountSumTree`, and +//! the dual-axis `ProvableCountProvableSumTree`. Each contributes its +//! own `inner_node_type().cost()` and is summed into both numerator +//! and denominator alongside the legacy four. + +use crate::{ + error::Error, estimated_costs::average_case_costs::EstimatedSumTrees, tree_type::TreeType, +}; + +impl EstimatedSumTrees { + pub(super) fn estimated_size_v2(&self) -> Result { + match self { + EstimatedSumTrees::NoSumTrees => Ok(0), + EstimatedSumTrees::SomeSumTrees { + sum_trees_weight, + big_sum_trees_weight, + count_trees_weight, + count_sum_trees_weight, + non_sum_trees_weight, + provable_sum_trees_weight, + provable_count_trees_weight, + provable_count_sum_trees_weight, + provable_count_provable_sum_trees_weight, + } => { + let total_weight = (*sum_trees_weight as u32) + .checked_add(*big_sum_trees_weight as u32) + .and_then(|w| w.checked_add(*count_trees_weight as u32)) + .and_then(|w| w.checked_add(*count_sum_trees_weight as u32)) + .and_then(|w| w.checked_add(*non_sum_trees_weight as u32)) + .and_then(|w| w.checked_add(*provable_sum_trees_weight as u32)) + .and_then(|w| w.checked_add(*provable_count_trees_weight as u32)) + .and_then(|w| w.checked_add(*provable_count_sum_trees_weight as u32)) + .and_then(|w| w.checked_add(*provable_count_provable_sum_trees_weight as u32)) + .ok_or(Error::Overflow( + "Estimated size total weight calculation overflowed", + ))?; + if total_weight == 0 { + return Err(Error::DivideByZero("weights add up to 0")); + } + let estimated_size = (*sum_trees_weight as u32 + * TreeType::SumTree.inner_node_type().cost()) + .checked_add( + *big_sum_trees_weight as u32 * TreeType::BigSumTree.inner_node_type().cost(), + ) + .and_then(|sum| { + sum.checked_add( + *count_trees_weight as u32 * TreeType::CountTree.inner_node_type().cost(), + ) + }) + .and_then(|sum| { + sum.checked_add( + *count_sum_trees_weight as u32 + * TreeType::CountSumTree.inner_node_type().cost(), + ) + }) + .and_then(|sum| { + sum.checked_add( + *provable_sum_trees_weight as u32 + * TreeType::ProvableSumTree.inner_node_type().cost(), + ) + }) + .and_then(|sum| { + sum.checked_add( + *provable_count_trees_weight as u32 + * TreeType::ProvableCountTree.inner_node_type().cost(), + ) + }) + .and_then(|sum| { + sum.checked_add( + *provable_count_sum_trees_weight as u32 + * TreeType::ProvableCountSumTree.inner_node_type().cost(), + ) + }) + .and_then(|sum| { + sum.checked_add( + *provable_count_provable_sum_trees_weight as u32 + * TreeType::ProvableCountProvableSumTree + .inner_node_type() + .cost(), + ) + }) + .ok_or(Error::Overflow("Estimated size calculation overflowed"))?; + + Ok(estimated_size / total_weight) + } + EstimatedSumTrees::AllSumTrees => Ok(TreeType::SumTree.inner_node_type().cost()), + EstimatedSumTrees::AllBigSumTrees => Ok(TreeType::BigSumTree.inner_node_type().cost()), + EstimatedSumTrees::AllCountTrees => Ok(TreeType::CountTree.inner_node_type().cost()), + EstimatedSumTrees::AllCountSumTrees => { + Ok(TreeType::CountSumTree.inner_node_type().cost()) + } + EstimatedSumTrees::AllProvableSumTrees => { + Ok(TreeType::ProvableSumTree.inner_node_type().cost()) + } + EstimatedSumTrees::AllProvableCountTrees => { + Ok(TreeType::ProvableCountTree.inner_node_type().cost()) + } + EstimatedSumTrees::AllProvableCountSumTrees => { + Ok(TreeType::ProvableCountSumTree.inner_node_type().cost()) + } + EstimatedSumTrees::AllProvableCountProvableSumTrees => { + Ok(TreeType::ProvableCountProvableSumTree + .inner_node_type() + .cost()) + } + } + } +} diff --git a/merk/src/estimated_costs/average_case_costs/mod.rs b/merk/src/estimated_costs/average_case_costs/mod.rs new file mode 100644 index 000000000..c681142f3 --- /dev/null +++ b/merk/src/estimated_costs/average_case_costs/mod.rs @@ -0,0 +1,1669 @@ +//! Average case costs for Merk + +// One submodule per versioned function, each laid out as +// `/{mod.rs, v0.rs, v1.rs, ...}` — `mod.rs` holds the +// version-dispatching wrapper and the `v*.rs` files hold the +// per-version implementations. +mod add_average_case_merk_propagate; +mod estimated_sum_trees_size; +mod value_with_feature_and_flags_size; + +pub use add_average_case_merk_propagate::{ + add_average_case_merk_propagate, average_case_merk_propagate, +}; + +use grovedb_costs::OperationCost; +use grovedb_version::version::GroveVersion; +use integer_encoding::VarInt; + +use crate::{ + error::Error, + tree::{kv::KV, Link, TreeNode}, + HASH_BLOCK_SIZE, HASH_BLOCK_SIZE_U32, HASH_LENGTH, HASH_LENGTH_U32, +}; +use crate::{merk::NodeType, tree_type::TreeType}; + +/// Average key size +pub type AverageKeySize = u8; +/// Average value size +pub type AverageValueSize = u32; +/// Average flags size +pub type AverageFlagsSize = u32; +/// Weight +pub type Weight = u8; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +/// Estimated number of sum trees +#[derive(Default)] +pub enum EstimatedSumTrees { + /// No sum trees + #[default] + NoSumTrees, + /// Some sum trees + SomeSumTrees { + /// Sum trees weight + sum_trees_weight: Weight, + /// Big Sum trees weight + big_sum_trees_weight: Weight, + /// Count trees weight + count_trees_weight: Weight, + /// Count Sum trees weight + count_sum_trees_weight: Weight, + /// Non sum trees weight + non_sum_trees_weight: Weight, + /// `ProvableSumTree` leaves. `ProvableSumNode` carries an i64 sum + /// that is baked into every node's hash; cost matches + /// `SumTree`'s `inner_node_type().cost()` (8 bytes). + provable_sum_trees_weight: Weight, + /// `ProvableCountTree` leaves. Same per-node cost as `CountTree` + /// (8 bytes), but the count is hash-committed. + provable_count_trees_weight: Weight, + /// `ProvableCountSumTree` leaves. Same per-node cost as + /// `CountSumTree` (16 bytes), but the count is hash-committed. + provable_count_sum_trees_weight: Weight, + /// `ProvableCountProvableSumTree` (PCPS) leaves — the v12 dual-axis + /// per-node variant. Same per-node cost as `CountSumTree` + /// (16 bytes), but BOTH the count and the sum are + /// hash-committed. + provable_count_provable_sum_trees_weight: Weight, + }, + /// All sum trees + AllSumTrees, + /// All big sum trees + AllBigSumTrees, + /// All count trees + AllCountTrees, + /// All count sum trees + AllCountSumTrees, + /// All `ProvableSumTree` leaves. + AllProvableSumTrees, + /// All `ProvableCountTree` leaves. + AllProvableCountTrees, + /// All `ProvableCountSumTree` leaves. + AllProvableCountSumTrees, + /// All `ProvableCountProvableSumTree` leaves. + AllProvableCountProvableSumTrees, +} + +// `EstimatedSumTrees::estimated_size` and its v0/v1/v2 implementations +// live in the `estimated_sum_trees_size` submodule (one v*.rs per +// version). + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +/// Estimated layer sizes +pub enum EstimatedLayerSizes { + /// All subtrees + AllSubtrees(AverageKeySize, EstimatedSumTrees, Option), + /// All items + AllItems(AverageKeySize, AverageValueSize, Option), + /// References + AllReference(AverageKeySize, AverageValueSize, Option), + /// Layer where every leaf is `Element::ItemWithSumItem`. Sizing adds + /// the i64 sum_value's worst-case varint encoding (`+10`) on top of + /// the plain-item layout — same constant the per-element helper + /// `Element::required_item_with_sum_item_space` uses. + AllItemsWithSumItem(AverageKeySize, AverageValueSize, Option), + /// Layer where every leaf is `Element::ReferenceWithSumItem`. Sizing + /// adds the same `+10` worst-case sum_value adjustment on top of the + /// plain-reference layout. + AllReferencesWithSumItem(AverageKeySize, AverageValueSize, Option), + /// Mix + Mix { + /// Subtrees size + subtrees_size: Option<( + AverageKeySize, + EstimatedSumTrees, + Option, + Weight, + )>, + /// Items size + items_size: Option<( + AverageKeySize, + AverageValueSize, + Option, + Weight, + )>, + /// References size + references_size: Option<( + AverageKeySize, + AverageValueSize, + Option, + Weight, + )>, + /// Weight of `ItemWithSumItem` leaves in the mix. Same sizing + /// formula as `items_size` plus `+10` for the worst-case i64 + /// sum_value varint. + items_with_sum_item_size: Option<( + AverageKeySize, + AverageValueSize, + Option, + Weight, + )>, + /// Weight of `ReferenceWithSumItem` leaves in the mix. Same + /// sizing formula as `references_size` plus `+10`. + references_with_sum_item_size: Option<( + AverageKeySize, + AverageValueSize, + Option, + Weight, + )>, + }, +} + +impl EstimatedLayerSizes { + /// Return average flags size for layer + pub fn layered_flags_size(&self) -> Result<&Option, Error> { + match self { + EstimatedLayerSizes::AllSubtrees(_, _, flags_size) => Ok(flags_size), + EstimatedLayerSizes::Mix { + subtrees_size: subtree_size, + .. + } => { + if let Some((_, _, flags_size, _)) = subtree_size { + Ok(flags_size) + } else { + Err(Error::WrongEstimatedCostsElementTypeForLevel( + "this mixed layer does not have costs for trees", + )) + } + } + _ => Err(Error::WrongEstimatedCostsElementTypeForLevel( + "this layer does not have costs for trees", + )), + } + } + + /// Returns the size of a subtree's feature and flags + /// This only takes into account subtrees in the estimated layer info + /// Only should be used when it is known to be a subtree + pub fn subtree_with_feature_and_flags_size( + &self, + grove_version: &GroveVersion, + ) -> Result { + match self { + EstimatedLayerSizes::AllSubtrees(_, estimated_sum_trees, flags_size) => { + // 1 for enum type + // 1 for empty + // 1 for flags size + Ok(estimated_sum_trees.estimated_size(grove_version)? + + flags_size.unwrap_or_default() + + 3) + } + EstimatedLayerSizes::Mix { subtrees_size, .. } => match subtrees_size { + None => Err(Error::WrongEstimatedCostsElementTypeForLevel( + "this layer is a mix but doesn't have subtrees", + )), + Some((_, est, fs, _)) => { + Ok(est.estimated_size(grove_version)? + fs.unwrap_or_default() + 3) + } + }, + _ => Err(Error::WrongEstimatedCostsElementTypeForLevel( + "this layer needs to have trees", + )), + } + } + + // `value_with_feature_and_flags_size` and its version-dispatched + // implementations now live in the + // `value_with_feature_and_flags_size` submodule. +} + +/// Approximate element count +pub type ApproximateElementCount = u32; +/// Estimated level number +pub type EstimatedLevelNumber = u32; +/// Estimated to be empty +pub type EstimatedToBeEmpty = bool; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +/// Information on an estimated layer +pub struct EstimatedLayerInformation { + /// The kind of tree we are in + pub tree_type: TreeType, + /// Estimated layer count + pub estimated_layer_count: EstimatedLayerCount, + /// Estimated layer sizes + pub estimated_layer_sizes: EstimatedLayerSizes, +} + +impl EstimatedLayerInformation {} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +/// Estimated elements and level number of a layer +pub enum EstimatedLayerCount { + /// Potentially at max elements + PotentiallyAtMaxElements, + /// Approximate elements + ApproximateElements(ApproximateElementCount), + /// Estimated level + EstimatedLevel(EstimatedLevelNumber, EstimatedToBeEmpty), +} + +impl EstimatedLayerCount { + /// Returns true if the tree is estimated to be empty. + pub fn estimated_to_be_empty(&self) -> bool { + match self { + EstimatedLayerCount::ApproximateElements(count) => *count == 0, + EstimatedLayerCount::PotentiallyAtMaxElements => false, + EstimatedLayerCount::EstimatedLevel(_, empty) => *empty, + } + } + + /// Estimate the number of levels based on the size of the tree, for big + /// trees this is very inaccurate. + pub fn estimate_levels(&self) -> u32 { + match self { + EstimatedLayerCount::ApproximateElements(n) => { + if *n == u32::MAX { + 32 + } else { + ((n + 1) as f32).log2().ceil() as u32 + } + } + EstimatedLayerCount::PotentiallyAtMaxElements => 32, + EstimatedLayerCount::EstimatedLevel(n, _) => *n, + } + } +} + +impl TreeNode { + /// Return estimate of average encoded tree size + pub fn average_case_encoded_tree_size( + not_prefixed_key_len: u32, + estimated_element_size: u32, + node_type: NodeType, + ) -> u32 { + // two option values for the left and right link + // the actual left and right link encoding size + // the encoded kv node size + 2 + (2 * Link::encoded_link_size(not_prefixed_key_len, node_type)) + + KV::encoded_kv_node_size(estimated_element_size, node_type) + } +} + +/// Add worst case for getting a merk node +pub fn add_average_case_get_merk_node( + cost: &mut OperationCost, + not_prefixed_key_len: u32, + approximate_element_size: u32, + node_type: NodeType, +) -> Result<(), Error> { + // Worst case scenario, the element is not already in memory. + // One direct seek has to be performed to read the node from storage. + cost.seek_count += 1; + + // To write a node to disk, the left link, right link and kv nodes are encoded. + // worst case, the node has both the left and right link present. + cost.storage_loaded_bytes += TreeNode::average_case_encoded_tree_size( + not_prefixed_key_len, + approximate_element_size, + node_type, + ) as u64; + Ok(()) +} + +/// Add worst case for getting a merk tree +pub fn add_average_case_merk_has_value( + cost: &mut OperationCost, + not_prefixed_key_len: u32, + estimated_element_size: u32, +) { + cost.seek_count += 1; + cost.storage_loaded_bytes += (not_prefixed_key_len + estimated_element_size) as u64; +} + +/// Add worst case for insertion into merk +pub fn add_average_case_merk_replace_layered( + cost: &mut OperationCost, + key_len: u32, + value_len: u32, + node_type: NodeType, +) { + cost.seek_count += 1; + cost.storage_cost.replaced_bytes = + KV::layered_value_byte_cost_size_for_key_and_value_lengths(key_len, value_len, node_type); + + // first lets add the value hash + cost.hash_node_calls += 1 + ((value_len - 1) / HASH_BLOCK_SIZE_U32); + // then let's add the combine hash + cost.hash_node_calls += 1; + // then let's add the kv_digest_to_kv_hash hash call + let hashed_size = key_len.encode_var_vec().len() as u32 + key_len + HASH_LENGTH_U32; + cost.hash_node_calls += 1 + ((hashed_size - 1) / HASH_BLOCK_SIZE_U32); + // then let's add the two block hashes for the node hash call + cost.hash_node_calls += 2; +} + +/// Add average case for deletion from merk +pub fn add_average_case_merk_delete_layered( + cost: &mut OperationCost, + _key_len: u32, + value_len: u32, +) { + // todo: verify this + cost.seek_count += 1; + cost.hash_node_calls += 1 + ((value_len - 1) / HASH_BLOCK_SIZE_U32); +} + +/// Add average case for deletion from merk +pub fn add_average_case_merk_delete(cost: &mut OperationCost, _key_len: u32, value_len: u32) { + // todo: verify this + cost.seek_count += 1; + cost.hash_node_calls += 1 + ((value_len - 1) / HASH_BLOCK_SIZE_U32); +} + +const fn node_hash_update_count() -> u32 { + // It's a hash of node hash, left and right + let bytes = HASH_LENGTH * 3; + // todo: verify this + + 1 + ((bytes - 1) / HASH_BLOCK_SIZE) as u32 +} + +/// Add worst case for getting a merk tree root hash +pub fn add_average_case_merk_root_hash(cost: &mut OperationCost) { + cost.hash_node_calls += node_hash_update_count(); +} + +#[cfg(test)] +mod tests { + use grovedb_costs::OperationCost; + use grovedb_version::version::GroveVersion; + + use super::*; + + #[test] + fn test_estimated_sum_trees_divide_by_zero() { + let estimated = EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 0, + big_sum_trees_weight: 0, + count_trees_weight: 0, + count_sum_trees_weight: 0, + non_sum_trees_weight: 0, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, + }; + let err = estimated + .estimated_size(GroveVersion::latest()) + .unwrap_err(); + assert!(matches!(err, Error::DivideByZero("weights add up to 0"))); + } + + #[test] + fn test_estimated_sum_trees_v0_formula_path() { + let estimated = EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 1, + big_sum_trees_weight: 0, + count_trees_weight: 0, + count_sum_trees_weight: 0, + non_sum_trees_weight: 3, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, + }; + let size = estimated.estimated_size(GroveVersion::first()).unwrap(); + assert_eq!(size, 6); + } + + #[test] + fn test_layered_flags_size_errors() { + let layer = EstimatedLayerSizes::AllItems(4, 10, Some(1)); + let err = layer.layered_flags_size().unwrap_err(); + assert!(matches!( + err, + Error::WrongEstimatedCostsElementTypeForLevel( + "this layer does not have costs for trees" + ) + )); + } + + #[test] + fn test_subtree_with_feature_and_flags_size_mix_missing_subtrees() { + let layer = EstimatedLayerSizes::Mix { + subtrees_size: None, + items_size: Some((4, 10, Some(1), 1)), + references_size: None, + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }; + let err = layer + .subtree_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap_err(); + assert!(matches!( + err, + Error::WrongEstimatedCostsElementTypeForLevel( + "this layer is a mix but doesn't have subtrees" + ) + )); + } + + #[test] + fn test_value_with_feature_and_flags_size_mix_without_weights() { + let layer = EstimatedLayerSizes::Mix { + subtrees_size: None, + items_size: None, + references_size: None, + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }; + let err = layer + .value_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap_err(); + assert!(matches!( + err, + Error::WrongEstimatedCostsElementTypeForLevel( + "this layer is a mix and does not have items, refs or trees" + ) + )); + } + + #[test] + fn test_estimated_layer_count_helpers() { + assert!(EstimatedLayerCount::ApproximateElements(0).estimated_to_be_empty()); + assert_eq!( + EstimatedLayerCount::ApproximateElements(7).estimate_levels(), + 3 + ); + assert_eq!( + EstimatedLayerCount::PotentiallyAtMaxElements.estimate_levels(), + 32 + ); + assert_eq!( + EstimatedLayerCount::EstimatedLevel(5, false).estimate_levels(), + 5 + ); + } + + #[test] + fn test_add_average_case_merk_propagate_version_mismatch_error() { + let mut custom_version = GroveVersion::latest().clone(); + custom_version + .merk_versions + .average_case_costs + .add_average_case_merk_propagate = 99; + + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLayerCount::EstimatedLevel(1, false), + estimated_layer_sizes: EstimatedLayerSizes::AllItems(5, 20, None), + }; + + let mut cost = OperationCost::default(); + let err = + add_average_case_merk_propagate(&mut cost, &layer_info, &custom_version).unwrap_err(); + assert!(matches!( + err, + Error::VersionError( + grovedb_version::error::GroveVersionError::UnknownVersionMismatch { .. } + ) + )); + } + + #[test] + fn test_add_average_case_merk_propagate_all_items_updates_cost() { + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), + estimated_layer_sizes: EstimatedLayerSizes::AllItems(8, 32, Some(2)), + }; + + let mut cost = OperationCost::default(); + add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::latest()).unwrap(); + + assert!(cost.seek_count > 0); + assert!(cost.hash_node_calls > 0); + assert!(cost.storage_cost.replaced_bytes > 0); + assert!(cost.storage_loaded_bytes > 0); + } + + // ========================================================================= + // v0 propagation (covers add_average_case_merk_propagate_v0 entirely) + // ========================================================================= + + #[test] + fn test_propagate_v0_all_subtrees() { + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), + estimated_layer_sizes: EstimatedLayerSizes::AllSubtrees( + 8, + EstimatedSumTrees::NoSumTrees, + Some(4), + ), + }; + let mut cost = OperationCost::default(); + add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::first()).unwrap(); + assert!(cost.seek_count > 0); + assert!(cost.storage_cost.replaced_bytes > 0); + assert!(cost.storage_loaded_bytes > 0); + } + + #[test] + fn test_propagate_v0_all_items() { + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), + estimated_layer_sizes: EstimatedLayerSizes::AllItems(8, 32, Some(2)), + }; + let mut cost = OperationCost::default(); + add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::first()).unwrap(); + assert!(cost.seek_count > 0); + assert!(cost.storage_cost.replaced_bytes > 0); + assert!(cost.storage_loaded_bytes > 0); + } + + #[test] + fn test_propagate_v0_all_reference() { + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), + estimated_layer_sizes: EstimatedLayerSizes::AllReference(8, 24, Some(2)), + }; + let mut cost = OperationCost::default(); + add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::first()).unwrap(); + assert!(cost.seek_count > 0); + assert!(cost.storage_cost.replaced_bytes > 0); + assert!(cost.storage_loaded_bytes > 0); + } + + #[test] + fn test_propagate_v0_mix() { + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), + estimated_layer_sizes: EstimatedLayerSizes::Mix { + subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 2)), + items_size: Some((8, 32, Some(2), 3)), + references_size: Some((8, 24, Some(1), 1)), + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }, + }; + let mut cost = OperationCost::default(); + add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::first()).unwrap(); + assert!(cost.seek_count > 0); + assert!(cost.storage_cost.replaced_bytes > 0); + assert!(cost.storage_loaded_bytes > 0); + } + + // ========================================================================= + // v1 propagation branches not yet covered (Mix, AllReference) + // ========================================================================= + + #[test] + fn test_propagate_v1_all_reference() { + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), + estimated_layer_sizes: EstimatedLayerSizes::AllReference(8, 24, Some(2)), + }; + let mut cost = OperationCost::default(); + add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::latest()).unwrap(); + assert!(cost.seek_count > 0); + assert!(cost.storage_cost.replaced_bytes > 0); + assert!(cost.storage_loaded_bytes > 0); + } + + #[test] + fn test_propagate_v1_mix() { + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), + estimated_layer_sizes: EstimatedLayerSizes::Mix { + subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 2)), + items_size: Some((8, 32, Some(2), 3)), + references_size: Some((8, 24, Some(1), 1)), + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }, + }; + let mut cost = OperationCost::default(); + add_average_case_merk_propagate(&mut cost, &layer_info, GroveVersion::latest()).unwrap(); + assert!(cost.seek_count > 0); + assert!(cost.storage_cost.replaced_bytes > 0); + assert!(cost.storage_loaded_bytes > 0); + } + + // ========================================================================= + // Utility functions with zero coverage + // ========================================================================= + + #[test] + fn test_add_average_case_merk_has_value() { + let mut cost = OperationCost::default(); + add_average_case_merk_has_value(&mut cost, 32, 100); + assert_eq!(cost.seek_count, 1); + assert_eq!(cost.storage_loaded_bytes, 132); + } + + #[test] + fn test_node_hash_update_count() { + let count = node_hash_update_count(); + // hash of 3 * HASH_LENGTH bytes: 1 + (bytes - 1) / HASH_BLOCK_SIZE + assert!(count > 0); + } + + #[test] + fn test_add_average_case_merk_root_hash() { + let mut cost = OperationCost::default(); + add_average_case_merk_root_hash(&mut cost); + assert!(cost.hash_node_calls > 0); + } + + // ========================================================================= + // EstimatedSumTrees variant arms not yet covered + // ========================================================================= + + #[test] + fn test_estimated_sum_trees_all_variant_arms() { + let v = GroveVersion::latest(); + let sum = EstimatedSumTrees::AllSumTrees.estimated_size(v).unwrap(); + assert!(sum > 0); + let big = EstimatedSumTrees::AllBigSumTrees.estimated_size(v).unwrap(); + assert!(big > 0); + let count = EstimatedSumTrees::AllCountTrees.estimated_size(v).unwrap(); + assert!(count > 0); + let count_sum = EstimatedSumTrees::AllCountSumTrees + .estimated_size(v) + .unwrap(); + assert!(count_sum > 0); + } + + // ========================================================================= + // EstimatedLayerCount edge case + // ========================================================================= + + #[test] + fn test_estimate_levels_u32_max() { + assert_eq!( + EstimatedLayerCount::ApproximateElements(u32::MAX).estimate_levels(), + 32 + ); + } + + // ========================================================================= + // EstimatedLayerSizes method branches + // ========================================================================= + + #[test] + fn test_layered_flags_size_mix_with_subtrees() { + let layer = EstimatedLayerSizes::Mix { + subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 1)), + items_size: Some((8, 32, Some(2), 1)), + references_size: None, + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }; + let flags = layer.layered_flags_size().unwrap(); + assert_eq!(*flags, Some(4)); + } + + #[test] + fn test_layered_flags_size_mix_without_subtrees() { + let layer = EstimatedLayerSizes::Mix { + subtrees_size: None, + items_size: Some((8, 32, Some(2), 1)), + references_size: None, + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }; + assert!(layer.layered_flags_size().is_err()); + } + + #[test] + fn test_subtree_size_all_subtrees() { + let layer = EstimatedLayerSizes::AllSubtrees(8, EstimatedSumTrees::NoSumTrees, Some(4)); + let size = layer + .subtree_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + assert_eq!(size, 7); // NoSumTrees=0, flags=4, base=3 + } + + #[test] + fn test_subtree_size_mix_with_subtrees() { + let layer = EstimatedLayerSizes::Mix { + subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 1)), + items_size: None, + references_size: None, + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }; + let size = layer + .subtree_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + assert_eq!(size, 7); + } + + #[test] + fn test_subtree_size_non_subtree_errors() { + let layer = EstimatedLayerSizes::AllReference(8, 24, Some(2)); + assert!(layer + .subtree_with_feature_and_flags_size(GroveVersion::latest()) + .is_err()); + } + + #[test] + fn test_value_size_all_reference() { + let layer = EstimatedLayerSizes::AllReference(8, 24, Some(2)); + let size = layer + .value_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + assert_eq!(size, 24 + 2 + 5); // value + flags + 5 for refs + } + + #[test] + fn test_value_size_all_subtrees() { + let layer = EstimatedLayerSizes::AllSubtrees(8, EstimatedSumTrees::NoSumTrees, Some(4)); + let size = layer + .value_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + assert_eq!(size, 7); + } + + /// v1 (grove v3) returns the proper weighted average. + /// item_size=37 weight=3, ref_size=30 weight=1, subtree_size=7 weight=2. + /// Σ(size_i · weight_i) = 111 + 30 + 14 = 155; combined_weight = 6; + /// 155 / 6 = 25. + #[test] + fn test_value_size_mix_weighted_combination_v1() { + let layer = EstimatedLayerSizes::Mix { + subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 2)), + items_size: Some((8, 32, Some(2), 3)), + references_size: Some((8, 24, Some(1), 1)), + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }; + let size = layer + .value_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + assert_eq!(size, 25); + } + + /// v0 (grove v1/v2) keeps the legacy unweighted formula: + /// (item + ref + subtree) / combined_weight = (37+30+7)/6 = 12. + /// Regression pin so the consensus-locked grove v1/v2 dispatch + /// can't accidentally pick up the fixed formula. + #[test] + fn test_value_size_mix_weighted_combination_v0_legacy() { + let layer = EstimatedLayerSizes::Mix { + subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 2)), + items_size: Some((8, 32, Some(2), 3)), + references_size: Some((8, 24, Some(1), 1)), + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }; + let size = layer + .value_with_feature_and_flags_size(GroveVersion::first()) + .unwrap(); + assert_eq!(size, 12); + } + + #[test] + fn test_value_size_mix_only_subtrees() { + let layer = EstimatedLayerSizes::Mix { + subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 2)), + items_size: None, + references_size: None, + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }; + let size = layer + .value_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + assert_eq!(size, 7); + } + + #[test] + fn test_value_size_mix_only_refs() { + let layer = EstimatedLayerSizes::Mix { + subtrees_size: None, + items_size: None, + references_size: Some((8, 24, Some(1), 1)), + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }; + let size = layer + .value_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + assert_eq!(size, 24 + 1 + 5); + } + + #[test] + fn test_value_size_mix_only_items() { + let layer = EstimatedLayerSizes::Mix { + subtrees_size: None, + items_size: Some((8, 32, Some(2), 3)), + references_size: None, + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }; + let size = layer + .value_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + assert_eq!(size, 32 + 2 + 3); + } + + // ========================================================================= + // Sum-bearing layer / tree variants + // ========================================================================= + + /// `AllItemsWithSumItem` adds +10 over the plain-item layout for the + /// worst-case i64 sum_value varint, matching the per-element helper + /// `Element::required_item_with_sum_item_space`. + #[test] + fn test_value_size_all_items_with_sum_item_is_all_items_plus_ten() { + let key = 8; + let value = 32; + let flags = Some(2); + + let plain = EstimatedLayerSizes::AllItems(key, value, flags); + let plain_size = plain + .value_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + + let with_sum = EstimatedLayerSizes::AllItemsWithSumItem(key, value, flags); + let with_sum_size = with_sum + .value_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + + assert_eq!(with_sum_size, plain_size + 10); + } + + /// `AllReferencesWithSumItem` adds the same +10 over the plain + /// `AllReference` formula. + #[test] + fn test_value_size_all_references_with_sum_item_is_all_reference_plus_ten() { + let key = 8; + let value = 24; + let flags = Some(1); + + let plain = EstimatedLayerSizes::AllReference(key, value, flags); + let plain_size = plain + .value_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + + let with_sum = EstimatedLayerSizes::AllReferencesWithSumItem(key, value, flags); + let with_sum_size = with_sum + .value_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + + assert_eq!(with_sum_size, plain_size + 10); + } + + /// `AllItemsWithSumItem` propagation cost strictly exceeds the plain + /// `AllItems` cost for the same key/value/flags. The +10 per-element + /// sum-value adjustment flows through both `storage_cost.replaced_bytes` + /// and `storage_loaded_bytes`. + #[test] + fn test_propagate_all_items_with_sum_item_exceeds_all_items() { + let layer_count = EstimatedLayerCount::EstimatedLevel(3, false); + let plain_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: EstimatedLayerSizes::AllItems(8, 32, Some(2)), + }; + let sum_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: EstimatedLayerSizes::AllItemsWithSumItem(8, 32, Some(2)), + }; + + let mut plain_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut plain_cost, &plain_info, GroveVersion::latest()) + .unwrap(); + let mut sum_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut sum_cost, &sum_info, GroveVersion::latest()).unwrap(); + + assert!( + sum_cost.storage_cost.replaced_bytes > plain_cost.storage_cost.replaced_bytes, + "sum-item replaced_bytes ({}) must exceed plain ({})", + sum_cost.storage_cost.replaced_bytes, + plain_cost.storage_cost.replaced_bytes, + ); + assert!( + sum_cost.storage_loaded_bytes > plain_cost.storage_loaded_bytes, + "sum-item storage_loaded_bytes ({}) must exceed plain ({})", + sum_cost.storage_loaded_bytes, + plain_cost.storage_loaded_bytes, + ); + } + + /// `AllReferencesWithSumItem` propagation cost strictly exceeds the + /// plain `AllReference` cost. + #[test] + fn test_propagate_all_references_with_sum_item_exceeds_all_reference() { + let layer_count = EstimatedLayerCount::EstimatedLevel(3, false); + let plain_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: EstimatedLayerSizes::AllReference(8, 24, Some(1)), + }; + let sum_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: EstimatedLayerSizes::AllReferencesWithSumItem(8, 24, Some(1)), + }; + + let mut plain_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut plain_cost, &plain_info, GroveVersion::latest()) + .unwrap(); + let mut sum_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut sum_cost, &sum_info, GroveVersion::latest()).unwrap(); + + assert!(sum_cost.storage_cost.replaced_bytes > plain_cost.storage_cost.replaced_bytes); + assert!(sum_cost.storage_loaded_bytes > plain_cost.storage_loaded_bytes); + } + + /// Mix with the new sum-bearing fields propagates without panicking + /// and produces a strictly larger cost than the same Mix with + /// `items_with_sum_item_size: None`. + #[test] + fn test_propagate_mix_with_items_with_sum_item_increases_cost() { + let layer_count = EstimatedLayerCount::EstimatedLevel(3, false); + let base = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: EstimatedLayerSizes::Mix { + subtrees_size: None, + items_size: Some((8, 32, Some(2), 1)), + references_size: None, + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }, + }; + let with_sum = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: EstimatedLayerSizes::Mix { + subtrees_size: None, + items_size: Some((8, 32, Some(2), 1)), + references_size: None, + items_with_sum_item_size: Some((8, 32, Some(2), 1)), + references_with_sum_item_size: None, + }, + }; + + let mut base_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut base_cost, &base, GroveVersion::latest()).unwrap(); + let mut sum_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut sum_cost, &with_sum, GroveVersion::latest()).unwrap(); + + // Adding a sum-item-bearing kind to the mix must push the + // averaged cost upward (each sum-item leaf is +10 over a plain + // item leaf). + assert!(sum_cost.storage_cost.replaced_bytes > base_cost.storage_cost.replaced_bytes); + } + + /// `Mix` layered_flags_size still picks up the subtrees' flags when + /// only the new sum-item fields are also present — sanity for the + /// `..` destructure. + #[test] + fn test_layered_flags_size_mix_with_subtrees_and_sum_item_fields() { + let layer = EstimatedLayerSizes::Mix { + subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(7), 1)), + items_size: None, + references_size: None, + items_with_sum_item_size: Some((8, 32, Some(2), 1)), + references_with_sum_item_size: None, + }; + let flags = layer.layered_flags_size().unwrap(); + assert_eq!(*flags, Some(7)); + } + + /// Regression: `Mix` with weights only on `items_with_sum_item_size` + /// short-circuits to the per-element sum-item size instead of + /// dividing by zero across the other empty kinds. + #[test] + fn test_value_size_mix_only_items_with_sum_item() { + let layer = EstimatedLayerSizes::Mix { + subtrees_size: None, + items_size: None, + references_size: None, + items_with_sum_item_size: Some((8, 32, Some(2), 3)), + references_with_sum_item_size: None, + }; + let size = layer + .value_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + // 32 (value) + 2 (flags) + 3 (item base) + 10 (sum varint worst case) + assert_eq!(size, 32 + 2 + 13); + } + + /// Regression: `Mix` with weights only on `references_with_sum_item_size`. + #[test] + fn test_value_size_mix_only_references_with_sum_item() { + let layer = EstimatedLayerSizes::Mix { + subtrees_size: None, + items_size: None, + references_size: None, + items_with_sum_item_size: None, + references_with_sum_item_size: Some((8, 24, Some(1), 2)), + }; + let size = layer + .value_with_feature_and_flags_size(GroveVersion::latest()) + .unwrap(); + // 24 + 1 + 5 (ref base) + 10 (sum varint) + assert_eq!(size, 24 + 1 + 15); + } + + // ========================================================================= + // EstimatedSumTrees v0/v1/v2 formula stability + new weights + // ========================================================================= + + /// New homogeneous variants resolve to their tree-type's + /// `inner_node_type().cost()`, mirroring the existing + /// `AllSumTrees`/`AllCountTrees` etc. + #[test] + fn test_estimated_sum_trees_all_provable_variant_arms() { + let v = GroveVersion::latest(); + assert_eq!( + EstimatedSumTrees::AllProvableSumTrees + .estimated_size(v) + .unwrap(), + TreeType::ProvableSumTree.inner_node_type().cost(), + ); + assert_eq!( + EstimatedSumTrees::AllProvableCountTrees + .estimated_size(v) + .unwrap(), + TreeType::ProvableCountTree.inner_node_type().cost(), + ); + assert_eq!( + EstimatedSumTrees::AllProvableCountSumTrees + .estimated_size(v) + .unwrap(), + TreeType::ProvableCountSumTree.inner_node_type().cost(), + ); + assert_eq!( + EstimatedSumTrees::AllProvableCountProvableSumTrees + .estimated_size(v) + .unwrap(), + TreeType::ProvableCountProvableSumTree + .inner_node_type() + .cost(), + ); + } + + /// v0 formula output is pinned to its historical value so the v2 + /// expansion can't silently shift fees on already-shipped paths + /// (grove v1 uses v0 here). Same input as + /// `test_estimated_sum_trees_v0_formula_path` but explicit. + #[test] + fn test_estimated_sum_trees_v0_output_pinned_for_provable_weights() { + // v0 ignores ALL `provable_*` weights — populating them must + // produce the same output as leaving them at zero. + let baseline = EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 1, + big_sum_trees_weight: 0, + count_trees_weight: 0, + count_sum_trees_weight: 0, + non_sum_trees_weight: 3, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, + }; + let with_provable = EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 1, + big_sum_trees_weight: 0, + count_trees_weight: 0, + count_sum_trees_weight: 0, + non_sum_trees_weight: 3, + provable_sum_trees_weight: 5, + provable_count_trees_weight: 7, + provable_count_sum_trees_weight: 9, + provable_count_provable_sum_trees_weight: 11, + }; + + let v0 = GroveVersion::first(); + assert_eq!( + baseline.estimated_size(v0).unwrap(), + with_provable.estimated_size(v0).unwrap(), + "v0 must ignore new provable_* weights", + ); + } + + /// Same stability invariant for v1: production grove v2 uses v1 of + /// this method, so its output must not shift when callers fill new + /// `provable_*` fields. + #[test] + fn test_estimated_sum_trees_v1_output_pinned_for_provable_weights() { + let mut v1_version = GroveVersion::latest().clone(); + v1_version + .merk_versions + .average_case_costs + .sum_tree_estimated_size = 1; + + let baseline = EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 2, + big_sum_trees_weight: 1, + count_trees_weight: 1, + count_sum_trees_weight: 1, + non_sum_trees_weight: 3, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, + }; + let with_provable = EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 2, + big_sum_trees_weight: 1, + count_trees_weight: 1, + count_sum_trees_weight: 1, + non_sum_trees_weight: 3, + provable_sum_trees_weight: 4, + provable_count_trees_weight: 6, + provable_count_sum_trees_weight: 8, + provable_count_provable_sum_trees_weight: 10, + }; + + assert_eq!( + baseline.estimated_size(&v1_version).unwrap(), + with_provable.estimated_size(&v1_version).unwrap(), + "v1 must ignore new provable_* weights so production grove v2 stays stable", + ); + } + + /// v2 must actually use the new `provable_*` weights — toggling them + /// changes the output, unlike v0/v1. + #[test] + fn test_estimated_sum_trees_v2_includes_provable_weights() { + let v = GroveVersion::latest(); // v3 → v2 of this method + + let baseline = EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 1, + big_sum_trees_weight: 0, + count_trees_weight: 0, + count_sum_trees_weight: 0, + non_sum_trees_weight: 0, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, + }; + let with_pcps = EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 1, + big_sum_trees_weight: 0, + count_trees_weight: 0, + count_sum_trees_weight: 0, + non_sum_trees_weight: 0, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 3, // PCPS dominates + }; + + let baseline_size = baseline.estimated_size(v).unwrap(); + let with_pcps_size = with_pcps.estimated_size(v).unwrap(); + assert_ne!( + baseline_size, with_pcps_size, + "v2 formula must respond to provable_count_provable_sum_trees_weight", + ); + } + + /// v2 weighted average: a homogeneous SomeSumTrees with only one + /// `provable_*` weight populated equals that tree-type's cost, + /// matching the `AllProvable*` shortcuts. + #[test] + fn test_estimated_sum_trees_v2_homogeneous_provable_matches_all_shortcut() { + let v = GroveVersion::latest(); + + let cases = [ + ( + EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 0, + big_sum_trees_weight: 0, + count_trees_weight: 0, + count_sum_trees_weight: 0, + non_sum_trees_weight: 0, + provable_sum_trees_weight: 4, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, + }, + EstimatedSumTrees::AllProvableSumTrees, + ), + ( + EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 0, + big_sum_trees_weight: 0, + count_trees_weight: 0, + count_sum_trees_weight: 0, + non_sum_trees_weight: 0, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 5, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, + }, + EstimatedSumTrees::AllProvableCountTrees, + ), + ( + EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 0, + big_sum_trees_weight: 0, + count_trees_weight: 0, + count_sum_trees_weight: 0, + non_sum_trees_weight: 0, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 6, + provable_count_provable_sum_trees_weight: 0, + }, + EstimatedSumTrees::AllProvableCountSumTrees, + ), + ( + EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 0, + big_sum_trees_weight: 0, + count_trees_weight: 0, + count_sum_trees_weight: 0, + non_sum_trees_weight: 0, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 7, + }, + EstimatedSumTrees::AllProvableCountProvableSumTrees, + ), + ]; + + for (homogeneous, shortcut) in cases { + assert_eq!( + homogeneous.estimated_size(v).unwrap(), + shortcut.estimated_size(v).unwrap(), + "homogeneous SomeSumTrees must match the All* shortcut for the same tree type", + ); + } + } + + /// v2 divide-by-zero guard kicks in even when only `provable_*` + /// weights are zero (legacy weights were zero too). + #[test] + fn test_estimated_sum_trees_v2_divide_by_zero() { + let v = GroveVersion::latest(); + let all_zero = EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 0, + big_sum_trees_weight: 0, + count_trees_weight: 0, + count_sum_trees_weight: 0, + non_sum_trees_weight: 0, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, + }; + let err = all_zero.estimated_size(v).unwrap_err(); + assert!(matches!(err, Error::DivideByZero("weights add up to 0"))); + } + + /// v0's divisor is `sum_trees + non_sum_trees`, not the full legacy + /// weight sum. If only `big_sum_trees_weight` (or any count weight) + /// is populated, the legacy sum is nonzero but the v0 denominator + /// is zero — the guard must surface `DivideByZero` rather than + /// panicking on `/ 0`. Regression for the v0 dispatch. + #[test] + fn test_estimated_sum_trees_v0_only_big_sum_weight_returns_divide_by_zero() { + let big_only = EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 0, + big_sum_trees_weight: 5, + count_trees_weight: 0, + count_sum_trees_weight: 0, + non_sum_trees_weight: 0, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, + }; + let err = big_only.estimated_size(GroveVersion::first()).unwrap_err(); + assert!(matches!(err, Error::DivideByZero("weights add up to 0"))); + } + + /// `check_grovedb_v0_v1_or_v2!` rejects any version outside {0, 1, 2}. + /// Exercises the version-gate error path on `estimated_size` so we + /// don't ship the macro without a regression test. + #[test] + fn test_estimated_sum_trees_unknown_version_error() { + let mut bad_version = GroveVersion::latest().clone(); + bad_version + .merk_versions + .average_case_costs + .sum_tree_estimated_size = 99; + let any = EstimatedSumTrees::SomeSumTrees { + sum_trees_weight: 1, + big_sum_trees_weight: 0, + count_trees_weight: 0, + count_sum_trees_weight: 0, + non_sum_trees_weight: 1, + provable_sum_trees_weight: 0, + provable_count_trees_weight: 0, + provable_count_sum_trees_weight: 0, + provable_count_provable_sum_trees_weight: 0, + }; + let err = any.estimated_size(&bad_version).unwrap_err(); + assert!(matches!( + err, + Error::VersionError( + grovedb_version::error::GroveVersionError::UnknownVersionMismatch { .. } + ) + )); + } + + // ========================================================================= + // v0 propagation (grove v1 path) with the new sum-bearing layer variants + // — the variants are new at the type level, so even on the v0 propagate + // dispatch they must compile and produce sensible costs. + // ========================================================================= + + #[test] + fn test_propagate_v0_all_items_with_sum_item() { + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), + estimated_layer_sizes: EstimatedLayerSizes::AllItemsWithSumItem(8, 32, Some(2)), + }; + let mut sum_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut sum_cost, &layer_info, GroveVersion::first()).unwrap(); + + // The +10 must flow through v0 propagate the same way as v1: the + // sum-bearing cost strictly exceeds the plain-AllItems cost. + let plain_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), + estimated_layer_sizes: EstimatedLayerSizes::AllItems(8, 32, Some(2)), + }; + let mut plain_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut plain_cost, &plain_info, GroveVersion::first()) + .unwrap(); + + assert!(sum_cost.storage_cost.replaced_bytes > plain_cost.storage_cost.replaced_bytes); + assert!(sum_cost.storage_loaded_bytes > plain_cost.storage_loaded_bytes); + } + + #[test] + fn test_propagate_v0_all_references_with_sum_item() { + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), + estimated_layer_sizes: EstimatedLayerSizes::AllReferencesWithSumItem(8, 24, Some(1)), + }; + let mut sum_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut sum_cost, &layer_info, GroveVersion::first()).unwrap(); + + let plain_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLayerCount::EstimatedLevel(3, false), + estimated_layer_sizes: EstimatedLayerSizes::AllReference(8, 24, Some(1)), + }; + let mut plain_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut plain_cost, &plain_info, GroveVersion::first()) + .unwrap(); + + assert!(sum_cost.storage_cost.replaced_bytes > plain_cost.storage_cost.replaced_bytes); + assert!(sum_cost.storage_loaded_bytes > plain_cost.storage_loaded_bytes); + } + + /// v0 propagate with `Mix` containing both sum-item fields populated + /// must produce a strictly larger cost than the same Mix with those + /// fields cleared. Exercises both the replaced_bytes and the + /// storage_loaded_bytes Mix arms in v0. + #[test] + fn test_propagate_v0_mix_with_both_sum_item_fields() { + let layer_count = EstimatedLayerCount::EstimatedLevel(3, false); + let base = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: EstimatedLayerSizes::Mix { + subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 1)), + items_size: Some((8, 32, Some(2), 1)), + references_size: Some((8, 24, Some(1), 1)), + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }, + }; + let with_sum = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: EstimatedLayerSizes::Mix { + subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 1)), + items_size: Some((8, 32, Some(2), 1)), + references_size: Some((8, 24, Some(1), 1)), + items_with_sum_item_size: Some((8, 32, Some(2), 1)), + references_with_sum_item_size: Some((8, 24, Some(1), 1)), + }, + }; + + let mut base_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut base_cost, &base, GroveVersion::first()).unwrap(); + let mut sum_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut sum_cost, &with_sum, GroveVersion::first()).unwrap(); + + assert!(sum_cost.storage_cost.replaced_bytes > base_cost.storage_cost.replaced_bytes); + assert!(sum_cost.storage_loaded_bytes > base_cost.storage_loaded_bytes); + } + + // ========================================================================= + // Propagate v1 vs v2 Mix-arm divergence (CodeRabbit's outside-diff finding). + // v1 (grove v2 dispatch): total_updates_cost / (nodes_updated · total_weight) + // — equivalent per-node avg / nodes_updated. Pre-existing buggy formula. + // v2 (grove v3 dispatch): nodes_updated · total_updates_cost / total_weight + // — proper layer total, matching the AllItems/AllSubtrees arms. + // ========================================================================= + + /// Build a v2-grove version (dispatches propagate v1, the legacy + /// formula) for regression pinning. Used only in tests so we don't + /// need to expose intermediate grove versions to consumers. + fn grove_version_dispatching_propagate_v1() -> GroveVersion { + let mut v = GroveVersion::latest().clone(); + v.merk_versions + .average_case_costs + .add_average_case_merk_propagate = 1; + v + } + + /// A Mix layer with only `items_size` populated is dimensionally + /// equivalent to `AllItems`. v2 brings the Mix cost up to match + /// `AllItems` (matching the documented invariant that the Mix + /// formula should reduce to the homogeneous case). v1's legacy + /// output is much smaller — pinned here so grove v2 fee outputs + /// don't drift. + #[test] + fn test_propagate_v2_mix_items_only_matches_all_items() { + let layer_count = EstimatedLayerCount::EstimatedLevel(3, false); + let mix = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: EstimatedLayerSizes::Mix { + subtrees_size: None, + items_size: Some((8, 32, Some(2), 1)), + references_size: None, + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }, + }; + let all_items = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: EstimatedLayerSizes::AllItems(8, 32, Some(2)), + }; + + let mut mix_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut mix_cost, &mix, GroveVersion::latest()).unwrap(); + let mut all_items_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut all_items_cost, &all_items, GroveVersion::latest()) + .unwrap(); + + assert_eq!( + mix_cost.storage_cost.replaced_bytes, all_items_cost.storage_cost.replaced_bytes, + "v2 Mix items-only should match AllItems exactly", + ); + assert_eq!( + mix_cost.storage_loaded_bytes, all_items_cost.storage_loaded_bytes, + "v2 Mix items-only should match AllItems exactly", + ); + } + + /// v2 Mix output is strictly larger than v1 Mix output for the same + /// input (because v1 divides by `nodes_updated²` worth of denominator). + #[test] + fn test_propagate_v2_mix_exceeds_v1_mix() { + let layer_count = EstimatedLayerCount::EstimatedLevel(3, false); + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: EstimatedLayerSizes::Mix { + subtrees_size: Some((8, EstimatedSumTrees::NoSumTrees, Some(4), 2)), + items_size: Some((8, 32, Some(2), 3)), + references_size: Some((8, 24, Some(1), 1)), + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }, + }; + + let mut v1_cost = OperationCost::default(); + add_average_case_merk_propagate( + &mut v1_cost, + &layer_info, + &grove_version_dispatching_propagate_v1(), + ) + .unwrap(); + let mut v2_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut v2_cost, &layer_info, GroveVersion::latest()).unwrap(); + + assert!( + v2_cost.storage_cost.replaced_bytes > v1_cost.storage_cost.replaced_bytes, + "v2 ({}) must exceed v1 ({}) for Mix replaced_bytes", + v2_cost.storage_cost.replaced_bytes, + v1_cost.storage_cost.replaced_bytes, + ); + assert!( + v2_cost.storage_loaded_bytes > v1_cost.storage_loaded_bytes, + "v2 ({}) must exceed v1 ({}) for Mix storage_loaded_bytes", + v2_cost.storage_loaded_bytes, + v1_cost.storage_loaded_bytes, + ); + } + + /// Non-Mix variants must produce identical output under v1 and v2 + /// (only the Mix arm changed). Sanity-check the dispatch isn't + /// changing anything outside the Mix path. + #[test] + fn test_propagate_v1_v2_agree_on_non_mix_variants() { + let layer_count = EstimatedLayerCount::EstimatedLevel(3, false); + let cases = [ + EstimatedLayerSizes::AllItems(8, 32, Some(2)), + EstimatedLayerSizes::AllReference(8, 24, Some(1)), + EstimatedLayerSizes::AllItemsWithSumItem(8, 32, Some(2)), + EstimatedLayerSizes::AllReferencesWithSumItem(8, 24, Some(1)), + EstimatedLayerSizes::AllSubtrees(8, EstimatedSumTrees::NoSumTrees, Some(4)), + ]; + + let v1_version = grove_version_dispatching_propagate_v1(); + for layer in cases { + let info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: layer, + }; + let mut v1 = OperationCost::default(); + add_average_case_merk_propagate(&mut v1, &info, &v1_version).unwrap(); + let mut v2 = OperationCost::default(); + add_average_case_merk_propagate(&mut v2, &info, GroveVersion::latest()).unwrap(); + assert_eq!( + v1.storage_cost.replaced_bytes, v2.storage_cost.replaced_bytes, + "non-Mix replaced_bytes must agree across propagate v1/v2 for {:?}", + info.estimated_layer_sizes, + ); + assert_eq!( + v1.storage_loaded_bytes, v2.storage_loaded_bytes, + "non-Mix storage_loaded_bytes must agree across propagate v1/v2 for {:?}", + info.estimated_layer_sizes, + ); + } + } + + /// `add_average_case_merk_propagate` rejects unknown versions + /// (regression for the v2 dispatch row). + #[test] + fn test_propagate_unknown_version_error_lists_all_three() { + let mut bad = GroveVersion::latest().clone(); + bad.merk_versions + .average_case_costs + .add_average_case_merk_propagate = 99; + let info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLayerCount::EstimatedLevel(2, false), + estimated_layer_sizes: EstimatedLayerSizes::AllItems(8, 32, None), + }; + let err = add_average_case_merk_propagate(&mut OperationCost::default(), &info, &bad) + .unwrap_err(); + match err { + Error::VersionError( + grovedb_version::error::GroveVersionError::UnknownVersionMismatch { + known_versions, + .. + }, + ) => assert_eq!(known_versions, vec![0, 1, 2]), + other => panic!("expected UnknownVersionMismatch, got {:?}", other), + } + } + + /// `value_with_feature_and_flags_size` dispatcher rejects unknown + /// versions — regression for the new dispatch path. + #[test] + fn test_value_with_feature_and_flags_size_unknown_version_error() { + let mut bad = GroveVersion::latest().clone(); + bad.merk_versions + .average_case_costs + .value_with_feature_and_flags_size = 99; + let layer = EstimatedLayerSizes::AllItems(8, 32, None); + let err = layer.value_with_feature_and_flags_size(&bad).unwrap_err(); + assert!(matches!( + err, + Error::VersionError( + grovedb_version::error::GroveVersionError::UnknownVersionMismatch { .. } + ) + )); + } + + /// Same as above but on the v1 propagate dispatch (grove v3 path), + /// so we cover the `references_with_sum_item_size` branch in the v1 + /// Mix arms — the items-only version of this test already covers + /// the items-with-sum branch. + #[test] + fn test_propagate_v1_mix_with_references_with_sum_item() { + let layer_count = EstimatedLayerCount::EstimatedLevel(3, false); + let base = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: EstimatedLayerSizes::Mix { + subtrees_size: None, + items_size: None, + references_size: Some((8, 24, Some(1), 1)), + items_with_sum_item_size: None, + references_with_sum_item_size: None, + }, + }; + let with_sum = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: layer_count, + estimated_layer_sizes: EstimatedLayerSizes::Mix { + subtrees_size: None, + items_size: None, + references_size: Some((8, 24, Some(1), 1)), + items_with_sum_item_size: None, + references_with_sum_item_size: Some((8, 24, Some(1), 1)), + }, + }; + + let mut base_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut base_cost, &base, GroveVersion::latest()).unwrap(); + let mut sum_cost = OperationCost::default(); + add_average_case_merk_propagate(&mut sum_cost, &with_sum, GroveVersion::latest()).unwrap(); + + assert!(sum_cost.storage_cost.replaced_bytes > base_cost.storage_cost.replaced_bytes); + assert!(sum_cost.storage_loaded_bytes > base_cost.storage_loaded_bytes); + } +} diff --git a/merk/src/estimated_costs/average_case_costs/value_with_feature_and_flags_size/mod.rs b/merk/src/estimated_costs/average_case_costs/value_with_feature_and_flags_size/mod.rs new file mode 100644 index 000000000..835d3517d --- /dev/null +++ b/merk/src/estimated_costs/average_case_costs/value_with_feature_and_flags_size/mod.rs @@ -0,0 +1,43 @@ +//! Versioned dispatch for +//! [`EstimatedLayerSizes::value_with_feature_and_flags_size`]. +//! +//! Each `v*.rs` adds one `value_with_feature_and_flags_size_v*` method via +//! a separate `impl EstimatedLayerSizes` block. The dispatcher below +//! selects the implementation by +//! `GroveVersion::merk_versions.average_case_costs.value_with_feature_and_flags_size`. + +mod v0; +mod v1; + +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; + +use super::EstimatedLayerSizes; +use crate::error::Error; + +impl EstimatedLayerSizes { + /// Returns the size of a value's feature and flags. Version-dispatched + /// to preserve consensus-locked grove v1/v2 outputs while letting + /// grove v3+ use a corrected weighted-average formula on the Mix + /// arm (v0 = `Σ size_i / Σ weight_i`, v1 = `Σ (size_i · weight_i) / Σ weight_i`). + /// Non-Mix variants are version-independent. + pub fn value_with_feature_and_flags_size( + &self, + grove_version: &GroveVersion, + ) -> Result { + match grove_version + .merk_versions + .average_case_costs + .value_with_feature_and_flags_size + { + 0 => self.value_with_feature_and_flags_size_v0(grove_version), + 1 => self.value_with_feature_and_flags_size_v1(grove_version), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "EstimatedLayerSizes::value_with_feature_and_flags_size".to_string(), + known_versions: vec![0, 1], + received: version, + }, + )), + } + } +} diff --git a/merk/src/estimated_costs/average_case_costs/value_with_feature_and_flags_size/v0.rs b/merk/src/estimated_costs/average_case_costs/value_with_feature_and_flags_size/v0.rs new file mode 100644 index 000000000..cc6bb61d2 --- /dev/null +++ b/merk/src/estimated_costs/average_case_costs/value_with_feature_and_flags_size/v0.rs @@ -0,0 +1,124 @@ +//! `value_with_feature_and_flags_size` v0 — legacy unweighted Mix +//! average. Pre-existing behavior used by grove v1/v2; bug-for-bug +//! compatible with shipped fee outputs. +//! +//! On a 3:1:2 item/ref/subtree mix this returns +//! `(item + ref + subtree) / 6`, ignoring weights in the numerator. + +use grovedb_version::version::GroveVersion; + +use crate::{error::Error, estimated_costs::average_case_costs::EstimatedLayerSizes}; + +impl EstimatedLayerSizes { + pub(super) fn value_with_feature_and_flags_size_v0( + &self, + grove_version: &GroveVersion, + ) -> Result { + match self { + EstimatedLayerSizes::AllItems(_, average_value_size, flags_size) => { + Ok(*average_value_size + flags_size.unwrap_or_default() + 3) + } + EstimatedLayerSizes::AllReference(_, average_value_size, flags_size) => { + Ok(*average_value_size + flags_size.unwrap_or_default() + 5) + } + EstimatedLayerSizes::AllItemsWithSumItem(_, average_value_size, flags_size) => { + Ok(*average_value_size + flags_size.unwrap_or_default() + 13) + } + EstimatedLayerSizes::AllReferencesWithSumItem(_, average_value_size, flags_size) => { + Ok(*average_value_size + flags_size.unwrap_or_default() + 15) + } + EstimatedLayerSizes::AllSubtrees(_, estimated_sum_trees, flags_size) => { + Ok(estimated_sum_trees.estimated_size(grove_version)? + + flags_size.unwrap_or_default() + + 3) + } + EstimatedLayerSizes::Mix { + subtrees_size, + items_size, + references_size, + items_with_sum_item_size, + references_with_sum_item_size, + } => { + let (item_size, item_weight) = items_size + .as_ref() + .map(|(_, vs, fs, weight)| (vs + fs.unwrap_or_default() + 3, *weight as u32)) + .unwrap_or_default(); + + let (ref_size, ref_weight) = references_size + .as_ref() + .map(|(_, vs, fs, weight)| (vs + fs.unwrap_or_default() + 5, *weight as u32)) + .unwrap_or_default(); + + let (item_sum_size, item_sum_weight) = items_with_sum_item_size + .as_ref() + .map(|(_, vs, fs, weight)| (vs + fs.unwrap_or_default() + 13, *weight as u32)) + .unwrap_or_default(); + + let (ref_sum_size, ref_sum_weight) = references_with_sum_item_size + .as_ref() + .map(|(_, vs, fs, weight)| (vs + fs.unwrap_or_default() + 15, *weight as u32)) + .unwrap_or_default(); + + let (subtree_size, subtree_weight) = match subtrees_size { + None => None, + Some((_, est, fs, weight)) => Some(( + est.estimated_size(grove_version)? + fs.unwrap_or_default() + 3, + *weight as u32, + )), + } + .unwrap_or_default(); + + let nonzero_kinds = (item_weight > 0) as u32 + + (ref_weight > 0) as u32 + + (subtree_weight > 0) as u32 + + (item_sum_weight > 0) as u32 + + (ref_sum_weight > 0) as u32; + if nonzero_kinds == 0 { + return Err(Error::WrongEstimatedCostsElementTypeForLevel( + "this layer is a mix and does not have items, refs or trees", + )); + } + if nonzero_kinds == 1 { + if item_weight > 0 { + return Ok(item_size); + } + if ref_weight > 0 { + return Ok(ref_size); + } + if subtree_weight > 0 { + return Ok(subtree_size); + } + if item_sum_weight > 0 { + return Ok(item_sum_size); + } + if ref_sum_weight > 0 { + return Ok(ref_sum_size); + } + } + + let combined_weight = item_weight + .checked_add(ref_weight) + .and_then(|a| a.checked_add(subtree_weight)) + .and_then(|a| a.checked_add(item_sum_weight)) + .and_then(|a| a.checked_add(ref_sum_weight)) + .ok_or(Error::Overflow("overflow for value size combining weights"))?; + + let combined_size = item_size + .checked_add(ref_size) + .and_then(|a| a.checked_add(subtree_size)) + .and_then(|a| a.checked_add(item_sum_size)) + .and_then(|a| a.checked_add(ref_sum_size)) + .ok_or(Error::Overflow("overflow for value size"))?; + + // `checked_div` only fails on divide-by-zero, which the + // `nonzero_kinds == 0` guard above already rules out; + // surface that case clearly anyway so the unreachable + // path produces an accurate error rather than the + // misleading "overflow" used previously. + combined_size + .checked_div(combined_weight) + .ok_or(Error::DivideByZero("value size divisor was zero")) + } + } + } +} diff --git a/merk/src/estimated_costs/average_case_costs/value_with_feature_and_flags_size/v1.rs b/merk/src/estimated_costs/average_case_costs/value_with_feature_and_flags_size/v1.rs new file mode 100644 index 000000000..631f8d7d8 --- /dev/null +++ b/merk/src/estimated_costs/average_case_costs/value_with_feature_and_flags_size/v1.rs @@ -0,0 +1,116 @@ +//! `value_with_feature_and_flags_size` v1 — proper weighted Mix +//! average: `Σ (size_i · weight_i) / Σ weight_i`. Non-Mix arms behave +//! identically to v0. + +use grovedb_version::version::GroveVersion; + +use crate::{error::Error, estimated_costs::average_case_costs::EstimatedLayerSizes}; + +impl EstimatedLayerSizes { + pub(super) fn value_with_feature_and_flags_size_v1( + &self, + grove_version: &GroveVersion, + ) -> Result { + match self { + EstimatedLayerSizes::AllItems(_, average_value_size, flags_size) => { + Ok(*average_value_size + flags_size.unwrap_or_default() + 3) + } + EstimatedLayerSizes::AllReference(_, average_value_size, flags_size) => { + Ok(*average_value_size + flags_size.unwrap_or_default() + 5) + } + EstimatedLayerSizes::AllItemsWithSumItem(_, average_value_size, flags_size) => { + Ok(*average_value_size + flags_size.unwrap_or_default() + 13) + } + EstimatedLayerSizes::AllReferencesWithSumItem(_, average_value_size, flags_size) => { + Ok(*average_value_size + flags_size.unwrap_or_default() + 15) + } + EstimatedLayerSizes::AllSubtrees(_, estimated_sum_trees, flags_size) => { + Ok(estimated_sum_trees.estimated_size(grove_version)? + + flags_size.unwrap_or_default() + + 3) + } + EstimatedLayerSizes::Mix { + subtrees_size, + items_size, + references_size, + items_with_sum_item_size, + references_with_sum_item_size, + } => { + let (item_size, item_weight) = items_size + .as_ref() + .map(|(_, vs, fs, weight)| (vs + fs.unwrap_or_default() + 3, *weight as u32)) + .unwrap_or_default(); + + let (ref_size, ref_weight) = references_size + .as_ref() + .map(|(_, vs, fs, weight)| (vs + fs.unwrap_or_default() + 5, *weight as u32)) + .unwrap_or_default(); + + let (item_sum_size, item_sum_weight) = items_with_sum_item_size + .as_ref() + .map(|(_, vs, fs, weight)| (vs + fs.unwrap_or_default() + 13, *weight as u32)) + .unwrap_or_default(); + + let (ref_sum_size, ref_sum_weight) = references_with_sum_item_size + .as_ref() + .map(|(_, vs, fs, weight)| (vs + fs.unwrap_or_default() + 15, *weight as u32)) + .unwrap_or_default(); + + let (subtree_size, subtree_weight) = match subtrees_size { + None => None, + Some((_, est, fs, weight)) => Some(( + est.estimated_size(grove_version)? + fs.unwrap_or_default() + 3, + *weight as u32, + )), + } + .unwrap_or_default(); + + let combined_weight = item_weight + .checked_add(ref_weight) + .and_then(|a| a.checked_add(subtree_weight)) + .and_then(|a| a.checked_add(item_sum_weight)) + .and_then(|a| a.checked_add(ref_sum_weight)) + .ok_or(Error::Overflow("overflow for value size combining weights"))?; + + if combined_weight == 0 { + return Err(Error::WrongEstimatedCostsElementTypeForLevel( + "this layer is a mix and does not have items, refs or trees", + )); + } + + // Proper weighted average. Numerator stays in u64 to + // tolerate large per-element costs × weights without + // overflowing the u32 size domain prematurely. + let weighted_sum = (item_size as u64) + .checked_mul(item_weight as u64) + .and_then(|a| { + (ref_size as u64) + .checked_mul(ref_weight as u64) + .and_then(|b| a.checked_add(b)) + }) + .and_then(|a| { + (subtree_size as u64) + .checked_mul(subtree_weight as u64) + .and_then(|b| a.checked_add(b)) + }) + .and_then(|a| { + (item_sum_size as u64) + .checked_mul(item_sum_weight as u64) + .and_then(|b| a.checked_add(b)) + }) + .and_then(|a| { + (ref_sum_size as u64) + .checked_mul(ref_sum_weight as u64) + .and_then(|b| a.checked_add(b)) + }) + .ok_or(Error::Overflow("overflow for weighted value size"))?; + + let result = weighted_sum / combined_weight as u64; + if result > u32::MAX as u64 { + return Err(Error::Overflow("weighted value size overflows u32")); + } + Ok(result as u32) + } + } + } +}