diff --git a/grovedb-element/src/element/constructor.rs b/grovedb-element/src/element/constructor.rs index fc97b5b55..d18bb3d4f 100644 --- a/grovedb-element/src/element/constructor.rs +++ b/grovedb-element/src/element/constructor.rs @@ -122,6 +122,45 @@ impl Element { Element::Reference(reference_path, max_reference_hop, flags) } + /// Set element to a reference-with-sum-item without flags or max hops. + /// + /// `sum_value` is the explicit weight that propagates to a sum-bearing + /// parent — independent of whatever the reference resolves to. + pub fn new_reference_with_sum_item( + reference_path: ReferencePathType, + sum_value: SumValue, + ) -> Self { + Element::ReferenceWithSumItem(reference_path, None, sum_value, None) + } + + /// Set element to a reference-with-sum-item with flags. + pub fn new_reference_with_sum_item_with_flags( + reference_path: ReferencePathType, + sum_value: SumValue, + flags: Option, + ) -> Self { + Element::ReferenceWithSumItem(reference_path, None, sum_value, flags) + } + + /// Set element to a reference-with-sum-item with max hops, no flags. + pub fn new_reference_with_sum_item_with_hops( + reference_path: ReferencePathType, + max_reference_hop: MaxReferenceHop, + sum_value: SumValue, + ) -> Self { + Element::ReferenceWithSumItem(reference_path, max_reference_hop, sum_value, None) + } + + /// Set element to a reference-with-sum-item with max hops and flags. + pub fn new_reference_with_sum_item_with_max_hops_and_flags( + reference_path: ReferencePathType, + max_reference_hop: MaxReferenceHop, + sum_value: SumValue, + flags: Option, + ) -> Self { + Element::ReferenceWithSumItem(reference_path, max_reference_hop, sum_value, flags) + } + /// Set element to a tree without flags pub fn new_tree(maybe_root_key: Option>) -> Self { Element::Tree(maybe_root_key, None) diff --git a/grovedb-element/src/element/helpers.rs b/grovedb-element/src/element/helpers.rs index 249efb01b..8dd7a1a4c 100644 --- a/grovedb-element/src/element/helpers.rs +++ b/grovedb-element/src/element/helpers.rs @@ -87,8 +87,10 @@ impl Element { /// /// `NonCounted` delegates to its inner element — sums still propagate /// when the wrapper is inserted into a sum-bearing parent. - /// `NotSummed` and `NotCountedOrSummed` return 0 — the wrappers' + /// `NotSummed` and `NotCountedOrSummed` return 0 — those wrappers' /// purpose is to contribute nothing to the parent sum tree. + /// `ReferenceWithSumItem` returns the explicit sum value carried on the + /// variant — independent of the resolved target's value. pub fn sum_value_or_default(&self) -> i64 { match self { Element::NonCounted(inner) => inner.sum_value_or_default(), @@ -97,7 +99,8 @@ impl Element { | Element::ItemWithSumItem(_, sum_value, _) | Element::SumTree(_, sum_value, _) | Element::CountSumTree(_, _, sum_value, _) - | Element::ProvableCountSumTree(_, _, sum_value, _) => *sum_value, + | Element::ProvableCountSumTree(_, _, sum_value, _) + | Element::ReferenceWithSumItem(_, _, sum_value, _) => *sum_value, _ => 0, } } @@ -128,6 +131,8 @@ impl Element { /// `NotSummed` returns `(inner_count, 0)` — sum is suppressed, count /// still propagates. /// `NotCountedOrSummed` returns `(0, 0)` — both are suppressed. + /// `ReferenceWithSumItem` returns `(1, sum_value)` — counts as one + /// element like a plain reference, contributes its explicit sum. pub fn count_sum_value_or_default(&self) -> (u64, i64) { match self { Element::NonCounted(inner) => (0, inner.sum_value_or_default()), @@ -135,7 +140,8 @@ impl Element { Element::NotCountedOrSummed(_) => (0, 0), Element::SumItem(sum_value, _) | Element::ItemWithSumItem(_, sum_value, _) - | Element::SumTree(_, sum_value, _) => (1, *sum_value), + | Element::SumTree(_, sum_value, _) + | Element::ReferenceWithSumItem(_, _, sum_value, _) => (1, *sum_value), Element::CountTree(_, count_value, _) => (*count_value, 0), Element::CountSumTree(_, count_value, sum_value, _) | Element::ProvableCountSumTree(_, count_value, sum_value, _) => { @@ -148,7 +154,8 @@ impl Element { /// Decoded the integer value in the SumItem element type, returns 0 for /// everything else. `NonCounted` delegates to its inner. `NotSummed` - /// and `NotCountedOrSummed` return 0. + /// and `NotCountedOrSummed` return 0. `ReferenceWithSumItem` returns + /// its explicit i64 sum cast to i128. pub fn big_sum_value_or_default(&self) -> i128 { match self { Element::NonCounted(inner) => inner.big_sum_value_or_default(), @@ -157,28 +164,33 @@ impl Element { | Element::ItemWithSumItem(_, sum_value, _) | Element::SumTree(_, sum_value, _) | Element::CountSumTree(_, _, sum_value, _) - | Element::ProvableCountSumTree(_, _, sum_value, _) => *sum_value as i128, + | Element::ProvableCountSumTree(_, _, sum_value, _) + | Element::ReferenceWithSumItem(_, _, sum_value, _) => *sum_value as i128, Element::BigSumTree(_, sum_value, _) => *sum_value, _ => 0, } } /// Decoded the integer value in the SumItem element type. Looks through - /// a `NonCounted` wrapper. + /// a `NonCounted` wrapper. Also returns the explicit sum from + /// `ReferenceWithSumItem`. pub fn as_sum_item_value(&self) -> Result { match self.underlying() { Element::SumItem(value, _) => Ok(*value), Element::ItemWithSumItem(_, value, _) => Ok(*value), + Element::ReferenceWithSumItem(_, _, value, _) => Ok(*value), _ => Err(ElementError::WrongElementType("expected a sum item")), } } /// Decoded the integer value in the SumItem element type. Looks through - /// a `NonCounted` wrapper. + /// a `NonCounted` wrapper. Also returns the explicit sum from + /// `ReferenceWithSumItem`. pub fn into_sum_item_value(self) -> Result { match self.into_underlying() { Element::SumItem(value, _) => Ok(value), Element::ItemWithSumItem(_, value, _) => Ok(value), + Element::ReferenceWithSumItem(_, _, value, _) => Ok(value), _ => Err(ElementError::WrongElementType("expected a sum item")), } } @@ -222,10 +234,12 @@ impl Element { } /// Gives the reference path type in the Reference element type. Looks - /// through a `NonCounted` wrapper. + /// through a `NonCounted` wrapper. Accepts both `Reference` and + /// `ReferenceWithSumItem`. pub fn into_reference_path_type(self) -> Result { match self.into_underlying() { Element::Reference(value, ..) => Ok(value), + Element::ReferenceWithSumItem(value, ..) => Ok(value), _ => Err(ElementError::WrongElementType("expected a reference")), } } @@ -362,9 +376,24 @@ impl Element { ) } - /// Check if the element is a reference. Looks through `NonCounted`. + /// Check if the element is a reference. Looks through `NonCounted`. Both + /// `Reference` and `ReferenceWithSumItem` are references — they share + /// the resolution path and combined-value-hash proof shape; the only + /// difference is that `ReferenceWithSumItem` carries an additional + /// `SumValue` that propagates into sum-bearing parents. pub fn is_reference(&self) -> bool { - matches!(self.underlying(), Element::Reference(..)) + matches!( + self.underlying(), + Element::Reference(..) | Element::ReferenceWithSumItem(..) + ) + } + + /// Check if the element is specifically a `ReferenceWithSumItem`. Looks + /// through `NonCounted`. Use `is_reference` when you only care that the + /// element is some kind of reference; use this when you need to + /// distinguish the sum-bearing variant. + pub fn is_reference_with_sum_item(&self) -> bool { + matches!(self.underlying(), Element::ReferenceWithSumItem(..)) } /// Check if the element is an item. Looks through `NonCounted`. @@ -421,7 +450,8 @@ impl Element { | Element::CommitmentTree(.., flags) | Element::MmrTree(.., flags) | Element::BulkAppendTree(.., flags) - | Element::DenseAppendOnlyFixedSizeTree(.., flags) => flags, + | Element::DenseAppendOnlyFixedSizeTree(.., flags) + | Element::ReferenceWithSumItem(.., flags) => flags, Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.get_flags(), @@ -446,7 +476,8 @@ impl Element { | Element::CommitmentTree(.., flags) | Element::MmrTree(.., flags) | Element::BulkAppendTree(.., flags) - | Element::DenseAppendOnlyFixedSizeTree(.., flags) => flags, + | Element::DenseAppendOnlyFixedSizeTree(.., flags) + | Element::ReferenceWithSumItem(.., flags) => flags, Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.get_flags_owned(), @@ -471,7 +502,8 @@ impl Element { | Element::CommitmentTree(.., flags) | Element::MmrTree(.., flags) | Element::BulkAppendTree(.., flags) - | Element::DenseAppendOnlyFixedSizeTree(.., flags) => flags, + | Element::DenseAppendOnlyFixedSizeTree(.., flags) + | Element::ReferenceWithSumItem(.., flags) => flags, Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.get_flags_mut(), @@ -496,7 +528,8 @@ impl Element { | Element::CommitmentTree(.., flags) | Element::MmrTree(.., flags) | Element::BulkAppendTree(.., flags) - | Element::DenseAppendOnlyFixedSizeTree(.., flags) => *flags = new_flags, + | Element::DenseAppendOnlyFixedSizeTree(.., flags) + | Element::ReferenceWithSumItem(.., flags) => *flags = new_flags, Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.set_flags(new_flags), @@ -545,6 +578,28 @@ impl Element { } } } + Element::ReferenceWithSumItem( + ref reference_path_type, + max_hop, + sum_value, + ref flags, + ) => { + match reference_path_type { + ReferencePathType::AbsolutePathReference(..) => self, + _ => { + // Mirror the Reference arm: rebuild as absolute, + // preserving the sum value. + let absolute_path = + path_from_reference_path_type(reference_path_type.clone(), path, key)?; + Element::ReferenceWithSumItem( + ReferencePathType::AbsolutePathReference(absolute_path), + max_hop, + sum_value, + flags.clone(), + ) + } + } + } Element::NonCounted(inner) => Element::NonCounted(Box::new( inner.convert_if_reference_to_absolute_reference(path, key)?, )), diff --git a/grovedb-element/src/element/mod.rs b/grovedb-element/src/element/mod.rs index 48f0b9739..4d6ead51a 100644 --- a/grovedb-element/src/element/mod.rs +++ b/grovedb-element/src/element/mod.rs @@ -178,6 +178,35 @@ pub enum Element { /// - A `NotCountedOrSummed` may not wrap any other wrapper or any /// non-tree element. NotCountedOrSummed(Box), + /// A reference that simultaneously carries an explicit `SumValue`. + /// + /// Resolves like `Element::Reference` on `get()` / `follow_reference()` + /// (hop-limited, cycle-detected, combined value hash) AND contributes + /// `sum_value` to any sum-bearing parent tree like `Element::SumItem` / + /// `Element::ItemWithSumItem`. The `sum_value` is **independent of the + /// resolved target's value** — it is the caller-supplied weight or amount + /// associated with the link itself. + /// + /// Use case: ranked / sortable index entries where the key encodes the + /// rank, the reference points to a canonical record elsewhere, and the + /// sum is the entry's monetary weight that aggregates into the parent's + /// total. + /// + /// May be inserted into any parent tree type. In non-sum parents (Tree, + /// CountTree, ProvableCountTree) the `sum_value` is silently ignored, + /// the same rule `Element::ItemWithSumItem` follows. + /// + /// Wrapper compatibility: + /// - **May** be wrapped in `NonCounted` to opt out of count propagation. + /// - **May NOT** be wrapped in `NotSummed` or `NotCountedOrSummed` — + /// those whitelists accept only the four sum-tree variants, not + /// item-like or reference-like base variants. + ReferenceWithSumItem( + ReferencePathType, + MaxReferenceHop, + SumValue, + Option, + ), } pub fn hex_to_ascii(hex_value: &[u8]) -> String { @@ -372,6 +401,18 @@ impl fmt::Display for Element { Element::NotCountedOrSummed(inner) => { write!(f, "NotCountedOrSummed({})", inner) } + Element::ReferenceWithSumItem(path, max_hop, sum_value, flags) => { + write!( + f, + "ReferenceWithSumItem({}, max_hop: {}, sum: {}{})", + path, + max_hop.map_or("None".to_string(), |h| h.to_string()), + sum_value, + flags + .as_ref() + .map_or(String::new(), |f| format!(", flags: {:?}", f)) + ) + } } } } @@ -400,6 +441,7 @@ impl Element { Element::MmrTree(..) => ElementType::MmrTree, Element::BulkAppendTree(..) => ElementType::BulkAppendTree, Element::DenseAppendOnlyFixedSizeTree(..) => ElementType::DenseAppendOnlyFixedSizeTree, + Element::ReferenceWithSumItem(..) => ElementType::ReferenceWithSumItem, Element::NonCounted(inner) => match inner.element_type() { ElementType::Item => ElementType::NonCountedItem, ElementType::Reference => ElementType::NonCountedReference, @@ -418,6 +460,7 @@ impl Element { ElementType::DenseAppendOnlyFixedSizeTree => { ElementType::NonCountedDenseAppendOnlyFixedSizeTree } + ElementType::ReferenceWithSumItem => ElementType::NonCountedReferenceWithSumItem, // Inner is always a base type — nested wrappers are // forbidden at construction and (de)serialization. already_non_counted => already_non_counted, @@ -570,6 +613,12 @@ mod serde_impl { NonCounted(Box), NotSummed(Box), NotCountedOrSummed(Box), + ReferenceWithSumItem( + ReferencePathType, + MaxReferenceHop, + SumValue, + Option, + ), } impl From for Element { @@ -603,6 +652,9 @@ mod serde_impl { ElementShadow::NotCountedOrSummed(inner) => { Element::NotCountedOrSummed(Box::new(Element::from(*inner))) } + ElementShadow::ReferenceWithSumItem(p, h, s, f) => { + Element::ReferenceWithSumItem(p, h, s, f) + } } } } diff --git a/grovedb-element/src/element/visualize.rs b/grovedb-element/src/element/visualize.rs index 26886154c..8d519fc05 100644 --- a/grovedb-element/src/element/visualize.rs +++ b/grovedb-element/src/element/visualize.rs @@ -191,6 +191,14 @@ impl Visualize for Element { drawer = inner.visualize(drawer)?; drawer.write(b")")?; } + Element::ReferenceWithSumItem(_ref, _max_hop, sum_value, flags) => { + drawer.write(format!("ref_with_sum_item: {sum_value}").as_bytes())?; + if let Some(f) = flags + && !f.is_empty() + { + drawer = f.visualize(drawer)?; + } + } } Ok(drawer) } diff --git a/grovedb-element/src/element_type.rs b/grovedb-element/src/element_type.rs index f1cbd9b9f..96f5fca51 100644 --- a/grovedb-element/src/element_type.rs +++ b/grovedb-element/src/element_type.rs @@ -33,8 +33,15 @@ pub const NOT_SUMMED_WRAPPER_DISCRIMINANT: u8 = 16; /// Twin-discriminant prefix for `NotSummedXxx` types: every twin is encoded /// as `NOT_SUMMED_TWIN_PREFIX | base`. The prefix has the high bit set /// (so all wrappers cluster in `0x80..` range) plus bits 4 and 5, which -/// distinguishes it from `NON_COUNTED_FLAG`'s `0x80` upper-nibble. Detection -/// is therefore an upper-nibble compare: `disc & 0xf0 == 0xb0`. +/// distinguishes it from `NON_COUNTED_FLAG`'s lower range. Detection is a +/// range check: `0x80..=0xaf` is NonCounted, `0xb0..=0xbf` is NotSummed. +/// +/// This places a soft cap on base discriminants: NonCounted bases can use +/// `0..=0x2F` (the low 6 bits of the twin), with bases `0..=0x0F` keeping +/// upper-nibble 0x80 and bases `0x10..=0x1F` falling under 0x90, `0x20..=0x2F` +/// under 0xA0. NotSummed bases must stay in `0..=0x0F` because the prefix +/// only leaves the low nibble. Today only four NotSummed bases are allowed +/// (4, 5, 7, 10) so this is not a constraint. pub const NOT_SUMMED_TWIN_PREFIX: u8 = 0xb0; /// Mask to recover the base type discriminant from a `NotSummedXxx` @@ -198,8 +205,15 @@ pub enum ElementType { BulkAppendTree = 13, /// Dense fixed-sized Merkle tree - discriminant 14 DenseAppendOnlyFixedSizeTree = 14, - // 15 is reserved as the on-disk wrapper byte and has no direct - // ElementType variant. + // 15 is reserved as the on-disk wrapper byte (NonCounted) and has no + // direct ElementType variant. + // 16 is reserved as the on-disk wrapper byte (NotSummed) and has no + // direct ElementType variant. + /// Reference that also carries an explicit `SumValue` - discriminant 18. + /// Resolves like `Reference` on `get()` and propagates `sum_value` into + /// sum-bearing parents like `SumItem` / `ItemWithSumItem`. See + /// `Element::ReferenceWithSumItem` for full semantics. + ReferenceWithSumItem = 18, /// Non-counted wrapper around `Item` - discriminant 128 NonCountedItem = 128, /// Non-counted wrapper around `Reference` - discriminant 129 @@ -230,6 +244,8 @@ pub enum ElementType { NonCountedBulkAppendTree = 141, /// Non-counted wrapper around `DenseAppendOnlyFixedSizeTree` - discriminant 142 NonCountedDenseAppendOnlyFixedSizeTree = 142, + /// Non-counted wrapper around `ReferenceWithSumItem` - discriminant 146 (`0x80 | 18`) + NonCountedReferenceWithSumItem = 146, /// Not-summed wrapper around `SumTree` - discriminant 180 (`0xb0 | 4`) NotSummedSumTree = 180, /// Not-summed wrapper around `BigSumTree` - discriminant 181 (`0xb0 | 5`) @@ -274,21 +290,22 @@ impl ElementType { "NonCounted wrapper has no inner element discriminant byte".to_string(), ) })?; - // The inner discriminant must be a base type — i.e. strictly less - // than NON_COUNTED_WRAPPER_DISCRIMINANT (15). Bytes 15+ are not - // valid on-disk inner discriminants: - // - 15 itself is the wrapper byte (nested wrappers forbidden), - // - 16 is the NotSummed wrapper byte (cross-nesting forbidden), - // - 17..=127 are unallocated, - // - 128..=142 are the synthetic NonCountedXxx twins which - // never appear on disk; without this check, the bitwise OR - // below would collapse `0x80 | inner_byte` into `inner_byte` - // and a payload like `[15, 128, ...]` would silently parse - // as `NonCountedItem`. - if inner_byte >= NON_COUNTED_WRAPPER_DISCRIMINANT { + // The inner discriminant must be a legal base type. Today those + // are `0..=14` plus `18` (ReferenceWithSumItem). Bytes 15, 16, + // and 17 are the wrapper bytes themselves (nested wrappers + // forbidden); 19..=127 are unallocated; 128..=142 + 146 are the + // synthetic NonCountedXxx twins which never appear on disk. + // Without this check, the bitwise OR below would collapse + // `0x80 | inner_byte` into `inner_byte` and a payload like + // `[15, 128, ...]` would silently parse as `NonCountedItem`. + // + // Use an explicit allowlist so the next base-variant addition is + // a one-line edit here and the check stays robust to new + // variants landing without updating the guard. + if !matches!(inner_byte, 0..=14 | 18) { return Err(ElementError::CorruptedData(format!( - "NonCounted inner discriminant must be a base type 0..={}, got {}", - NON_COUNTED_WRAPPER_DISCRIMINANT - 1, + "NonCounted inner discriminant must be a base type \ + (0..=14 or 18), got {}", inner_byte ))); } @@ -336,11 +353,14 @@ impl ElementType { } } - /// Returns true if this is a `NonCountedXxx` discriminant. Tested by - /// upper-nibble compare since `NotSummedXxx` also has bit 7 set. + /// Returns true if this is a `NonCountedXxx` discriminant. Range check: + /// `NonCountedXxx` lives in `0x80..=0xaf`, `NotSummedXxx` in + /// `0xb0..=0xbf`. Both ranges have bit 7 set but only NonCounted's upper + /// nibble stays below `0xb`. #[inline] pub const fn is_non_counted(self) -> bool { - (self as u8) & 0xf0 == NON_COUNTED_FLAG + let disc = self as u8; + disc >= NON_COUNTED_FLAG && disc < NOT_SUMMED_TWIN_PREFIX } /// Returns true if this is a `NotSummedXxx` discriminant. @@ -514,10 +534,15 @@ impl ElementType { } /// Returns true if this element type is a reference. Looks through the - /// `NonCounted` wrapper. + /// `NonCounted` wrapper. Both `Reference` and `ReferenceWithSumItem` are + /// references — they share the combined-value-hash proof shape and are + /// resolved by the same `follow_reference` chain. #[inline] pub fn is_reference(&self) -> bool { - matches!(self.base(), ElementType::Reference) + matches!( + self.base(), + ElementType::Reference | ElementType::ReferenceWithSumItem + ) } /// Returns true if this element type is any kind of item (not a tree or @@ -548,6 +573,7 @@ impl ElementType { ElementType::MmrTree => "mmr tree", ElementType::BulkAppendTree => "bulk_append_tree", ElementType::DenseAppendOnlyFixedSizeTree => "dense_tree", + ElementType::ReferenceWithSumItem => "reference with sum item", ElementType::NonCountedItem => "non_counted item", ElementType::NonCountedReference => "non_counted reference", ElementType::NonCountedTree => "non_counted tree", @@ -563,6 +589,7 @@ impl ElementType { ElementType::NonCountedMmrTree => "non_counted mmr tree", ElementType::NonCountedBulkAppendTree => "non_counted bulk_append_tree", ElementType::NonCountedDenseAppendOnlyFixedSizeTree => "non_counted dense_tree", + ElementType::NonCountedReferenceWithSumItem => "non_counted reference with sum item", ElementType::NotSummedSumTree => "not_summed sum tree", ElementType::NotSummedBigSumTree => "not_summed big sum tree", ElementType::NotSummedCountSumTree => "not_summed count sum tree", @@ -604,6 +631,9 @@ impl TryFrom for ElementType { 14 => Ok(ElementType::DenseAppendOnlyFixedSizeTree), // 15 is the raw NonCounted wrapper byte; from_serialized_value // resolves it by reading the inner discriminant. + // 16 is the raw NotSummed wrapper byte; same treatment. + // 17 is the raw NotCountedOrSummed wrapper byte; same treatment. + 18 => Ok(ElementType::ReferenceWithSumItem), 128 => Ok(ElementType::NonCountedItem), 129 => Ok(ElementType::NonCountedReference), 130 => Ok(ElementType::NonCountedTree), @@ -619,6 +649,7 @@ impl TryFrom for ElementType { 140 => Ok(ElementType::NonCountedMmrTree), 141 => Ok(ElementType::NonCountedBulkAppendTree), 142 => Ok(ElementType::NonCountedDenseAppendOnlyFixedSizeTree), + 146 => Ok(ElementType::NonCountedReferenceWithSumItem), 180 => Ok(ElementType::NotSummedSumTree), 181 => Ok(ElementType::NotSummedBigSumTree), 183 => Ok(ElementType::NotSummedCountSumTree), @@ -683,9 +714,21 @@ mod tests { // 15 is the raw NonCounted wrapper byte and is rejected by TryFrom; // it has no direct ElementType variant (use from_serialized_value). assert!(ElementType::try_from(15).is_err()); + // 16 is the raw NotSummed wrapper byte; same treatment. assert!(ElementType::try_from(16).is_err()); + // 17 is the raw NotCountedOrSummed wrapper byte; same treatment. + assert!(ElementType::try_from(17).is_err()); - // NonCounted twins (0x80 | base): 128..142 + // Base discriminant 18 is ReferenceWithSumItem. + assert_eq!( + ElementType::try_from(18).unwrap(), + ElementType::ReferenceWithSumItem + ); + // 19..=127 are unallocated and invalid. + assert!(ElementType::try_from(19).is_err()); + assert!(ElementType::try_from(100).is_err()); + + // NonCounted twins (0x80 | base): 128..142 plus 146 (twin of base 18). assert_eq!( ElementType::try_from(128).unwrap(), ElementType::NonCountedItem @@ -698,10 +741,20 @@ mod tests { ElementType::try_from(142).unwrap(), ElementType::NonCountedDenseAppendOnlyFixedSizeTree ); + assert_eq!( + ElementType::try_from(146).unwrap(), + ElementType::NonCountedReferenceWithSumItem + ); // Bytes between the base and NonCounted-twin ranges are invalid. assert!(ElementType::try_from(127).is_err()); - // Bytes between NonCounted-twin and NotSummed-twin ranges are invalid. + // 143 (= 0x80|15), 144 (= 0x80|16), 145 (= 0x80|17): wrapper bytes + // have no base, so these would synthesize a wrapper-on-wrapper twin. assert!(ElementType::try_from(143).is_err()); + assert!(ElementType::try_from(144).is_err()); + assert!(ElementType::try_from(145).is_err()); + // 147..=179 (between NonCounted-twin and NotSummed-twin ranges) are + // also invalid. + assert!(ElementType::try_from(147).is_err()); assert!(ElementType::try_from(179).is_err()); // NotSummed twins (0xb0 | base): only the four sum-tree bases @@ -910,6 +963,7 @@ mod tests { // Trees and references have combined hash assert!(ElementType::Reference.has_combined_value_hash()); + assert!(ElementType::ReferenceWithSumItem.has_combined_value_hash()); assert!(ElementType::Tree.has_combined_value_hash()); assert!(ElementType::SumTree.has_combined_value_hash()); assert!(ElementType::BigSumTree.has_combined_value_hash()); @@ -922,6 +976,7 @@ mod tests { assert!(ElementType::NonCountedSumItem.has_simple_value_hash()); assert!(ElementType::NonCountedTree.has_combined_value_hash()); assert!(ElementType::NonCountedReference.has_combined_value_hash()); + assert!(ElementType::NonCountedReferenceWithSumItem.has_combined_value_hash()); } #[test] @@ -945,6 +1000,15 @@ mod tests { ElementType::Reference.proof_node_type(None), ProofNodeType::KvRefValueHash ); + // ReferenceWithSumItem shares the reference proof shape. + assert_eq!( + ElementType::ReferenceWithSumItem.proof_node_type(None), + ProofNodeType::KvRefValueHash + ); + assert_eq!( + ElementType::ReferenceWithSumItem.proof_node_type(Some(ElementType::SumTree)), + ProofNodeType::KvRefValueHash + ); // Trees should use KvValueHash (verifier trusts hash) assert_eq!( @@ -999,6 +1063,12 @@ mod tests { ElementType::Reference.proof_node_type(pct), ProofNodeType::KvRefValueHashCount ); + // ReferenceWithSumItem shares the reference proof shape inside + // ProvableCountTree parents. + assert_eq!( + ElementType::ReferenceWithSumItem.proof_node_type(pct), + ProofNodeType::KvRefValueHashCount + ); // Subtrees use KvValueHashFeatureType (combined hash + count) assert_eq!( @@ -1123,10 +1193,19 @@ mod tests { // would silently parse as `NonCountedItem`. assert!(ElementType::from_serialized_value(&[15, 128]).is_err()); assert!(ElementType::from_serialized_value(&[15, 142]).is_err()); - // Wrapper with an unallocated mid-range inner byte (16..=127) is - // also rejected, even though it has no high bit set. + // Wrapper with an unallocated mid-range inner byte (16, 17, 19..=127) + // is also rejected, even though it has no high bit set. assert!(ElementType::from_serialized_value(&[15, 16]).is_err()); + assert!(ElementType::from_serialized_value(&[15, 17]).is_err()); + assert!(ElementType::from_serialized_value(&[15, 19]).is_err()); assert!(ElementType::from_serialized_value(&[15, 100]).is_err()); + + // Inner byte 18 (ReferenceWithSumItem) IS a legal base; resolves to + // the synthetic NonCountedReferenceWithSumItem twin. + assert_eq!( + ElementType::from_serialized_value(&[15, 18]).unwrap(), + ElementType::NonCountedReferenceWithSumItem + ); } #[test] @@ -1146,6 +1225,10 @@ mod tests { assert!(ElementType::MmrTree.is_tree()); assert!(ElementType::BulkAppendTree.is_tree()); assert!(ElementType::DenseAppendOnlyFixedSizeTree.is_tree()); + // ReferenceWithSumItem is a reference, not a tree and not an item. + assert!(!ElementType::ReferenceWithSumItem.is_tree()); + assert!(ElementType::ReferenceWithSumItem.is_reference()); + assert!(!ElementType::ReferenceWithSumItem.is_item()); // The wrapper is transparent: NonCountedTree is a tree, NonCountedItem is not. assert!(!ElementType::NonCountedItem.is_tree()); @@ -1159,6 +1242,8 @@ mod tests { assert!(ElementType::NonCountedItem.is_item()); assert!(ElementType::NonCountedSumItem.is_item()); assert!(ElementType::NonCountedReference.is_reference()); + assert!(ElementType::NonCountedReferenceWithSumItem.is_reference()); + assert!(!ElementType::NonCountedReferenceWithSumItem.is_item()); } /// Verifies that serialized Element discriminants match ElementType @@ -1259,13 +1344,27 @@ mod tests { ElementType::DenseAppendOnlyFixedSizeTree, "DenseAppendOnlyFixedSizeTree", ), + // discriminant 18 (15, 16, 17 are wrapper bytes — no base variants) + ( + Element::ReferenceWithSumItem( + ReferencePathType::AbsolutePathReference(vec![vec![1]]), + None, + 42, + None, + ), + ElementType::ReferenceWithSumItem, + "ReferenceWithSumItem", + ), ]; - // Verify we're testing all 15 base discriminants (0-14) + // Verify we're testing all 16 base discriminants: 0..=14 and 18. + // (15 = NonCounted wrapper byte, 16 = NotSummed wrapper byte, + // 17 = NotCountedOrSummed wrapper byte — none has a base + // ElementType variant.) assert_eq!( test_cases.len(), - 15, - "Expected 15 base Element variants in test, got {}", + 16, + "Expected 16 base Element variants in test, got {}", test_cases.len() ); @@ -1317,7 +1416,7 @@ mod tests { fn test_non_counted_wrapper_discriminant_pinned() { use grovedb_version::version::GroveVersion; - use crate::element::Element; + use crate::{element::Element, reference_path::ReferencePathType}; let grove_version = GroveVersion::latest(); @@ -1348,6 +1447,17 @@ mod tests { 8, "NonCounted(ProvableCountTree)", ), + ( + Element::NonCounted(Box::new(Element::ReferenceWithSumItem( + ReferencePathType::AbsolutePathReference(vec![vec![1]]), + None, + 42, + None, + ))), + ElementType::NonCountedReferenceWithSumItem, + 18, + "NonCounted(ReferenceWithSumItem)", + ), ]; for (element, expected_type, expected_inner_disc, name) in cases { @@ -1494,8 +1604,8 @@ mod tests { // NotSummed twins (180..186), NotCountedOrSummed twins (196..202), // and unallocated ranges. for bad in [ - 0u8, 1, 2, 3, 6, 8, 9, 11, 12, 13, 14, 15, 16, 17, 100, 128, 142, 180, 186, 196, 202, - 255, + 0u8, 1, 2, 3, 6, 8, 9, 11, 12, 13, 14, 15, 16, 17, 18, 100, 128, 142, 146, 180, 186, + 196, 202, 255, ] { assert!( ElementType::from_serialized_value(&[16, bad]).is_err(), @@ -1533,8 +1643,8 @@ mod tests { // All other inner bytes are rejected. for bad in [ - 0u8, 1, 2, 3, 6, 8, 9, 11, 12, 13, 14, 15, 16, 17, 100, 128, 142, 180, 186, 196, 202, - 255, + 0u8, 1, 2, 3, 6, 8, 9, 11, 12, 13, 14, 15, 16, 17, 18, 100, 128, 142, 146, 180, 186, + 196, 202, 255, ] { assert!( ElementType::from_serialized_value(&[17, bad]).is_err(), diff --git a/grovedb-element/tests/element_constructors_helpers.rs b/grovedb-element/tests/element_constructors_helpers.rs index 279ffe9ef..949544c40 100644 --- a/grovedb-element/tests/element_constructors_helpers.rs +++ b/grovedb-element/tests/element_constructors_helpers.rs @@ -536,3 +536,190 @@ fn convert_if_reference_to_absolute_reference_converts_and_preserves_other_types ElementError::InvalidInput("reference stored path cannot satisfy reference constraints") )); } + +#[test] +fn constructors_create_expected_reference_with_sum_item_variants() { + let ref_path = ReferencePathType::AbsolutePathReference(vec![b"a".to_vec(), b"b".to_vec()]); + + assert_eq!( + Element::new_reference_with_sum_item(ref_path.clone(), 42), + Element::ReferenceWithSumItem(ref_path.clone(), None, 42, None) + ); + assert_eq!( + Element::new_reference_with_sum_item_with_flags(ref_path.clone(), 42, sample_flags()), + Element::ReferenceWithSumItem(ref_path.clone(), None, 42, sample_flags()) + ); + assert_eq!( + Element::new_reference_with_sum_item_with_hops(ref_path.clone(), Some(7), 42), + Element::ReferenceWithSumItem(ref_path.clone(), Some(7), 42, None) + ); + assert_eq!( + Element::new_reference_with_sum_item_with_max_hops_and_flags( + ref_path.clone(), + Some(7), + 42, + sample_flags() + ), + Element::ReferenceWithSumItem(ref_path, Some(7), 42, sample_flags()) + ); +} + +#[test] +fn reference_with_sum_item_helpers_pass_through_sum_and_reference_predicates() { + let ref_path = ReferencePathType::SiblingReference(b"k".to_vec()); + let element = Element::new_reference_with_sum_item(ref_path.clone(), 42); + + // It is a reference, NOT an item — even though it carries a sum value. + assert!(element.is_reference()); + assert!(element.is_reference_with_sum_item()); + assert!(!element.is_any_item()); + assert!(!element.is_basic_item()); + assert!(!element.has_basic_item()); + assert!(!element.is_sum_item()); + assert!(!element.is_item_with_sum_item()); + assert!(!element.is_any_tree()); + + // Sum propagation: contributes its explicit value, independent of target. + assert_eq!(element.sum_value_or_default(), 42); + assert_eq!(element.big_sum_value_or_default(), 42); + // Counts as a single element. + assert_eq!(element.count_value_or_default(), 1); + assert_eq!(element.count_sum_value_or_default(), (1, 42)); + + // Negative sum is preserved. + let neg = Element::new_reference_with_sum_item(ref_path.clone(), -100); + assert_eq!(neg.sum_value_or_default(), -100); + assert_eq!(neg.count_sum_value_or_default(), (1, -100)); + + // `as_sum_item_value` extracts the sum even though this is not an item. + assert_eq!(element.as_sum_item_value().unwrap(), 42); + assert_eq!(element.clone().into_sum_item_value().unwrap(), 42); + + // Path round-trips through the reference accessor. + assert_eq!( + element.clone().into_reference_path_type().unwrap(), + ref_path + ); +} + +#[test] +fn non_counted_reference_with_sum_item_zeros_count_keeps_sum() { + let inner = Element::new_reference_with_sum_item( + ReferencePathType::AbsolutePathReference(vec![b"a".to_vec()]), + 50, + ); + let nc = Element::new_non_counted(inner.clone()).expect("wrap ok"); + assert!(nc.is_non_counted()); + // Count is suppressed; sum still propagates. + assert_eq!(nc.count_value_or_default(), 0); + assert_eq!(nc.sum_value_or_default(), 50); + assert_eq!(nc.count_sum_value_or_default(), (0, 50)); + // The reference predicate looks through the wrapper. + assert!(nc.is_reference()); + assert!(nc.is_reference_with_sum_item()); + // Underlying returns the inner. + assert_eq!(nc.underlying(), &inner); +} + +#[test] +fn not_summed_rejects_reference_with_sum_item() { + // NotSummed accepts only sum-tree variants; a reference-with-sum-item is + // not a tree. + let element = Element::new_reference_with_sum_item( + ReferencePathType::SiblingReference(b"k".to_vec()), + 10, + ); + assert!(Element::new_not_summed(element).is_err()); +} + +#[test] +fn convert_if_reference_to_absolute_reference_preserves_sum_value() { + let path = [b"root".as_ref(), b"branch".as_ref()]; + let key = Some(b"leaf".as_ref()); + + // Cousin reference with explicit sum gets converted to absolute, sum is + // preserved. + let cousin = Element::new_reference_with_sum_item_with_max_hops_and_flags( + ReferencePathType::CousinReference(b"other".to_vec()), + Some(3), + 77, + Some(vec![7]), + ); + let converted = cousin + .convert_if_reference_to_absolute_reference(&path, key) + .unwrap(); + assert_eq!( + converted, + Element::ReferenceWithSumItem( + ReferencePathType::AbsolutePathReference(vec![ + b"root".to_vec(), + b"other".to_vec(), + b"leaf".to_vec(), + ]), + Some(3), + 77, + Some(vec![7]), + ) + ); + + // Already-absolute variant is returned unchanged. + let absolute = Element::new_reference_with_sum_item( + ReferencePathType::AbsolutePathReference(vec![b"a".to_vec(), b"b".to_vec()]), + 15, + ); + assert_eq!( + absolute + .clone() + .convert_if_reference_to_absolute_reference(&path, key) + .unwrap(), + absolute + ); +} + +#[test] +fn flag_accessors_handle_reference_with_sum_item() { + let mut element = Element::new_reference_with_sum_item_with_flags( + ReferencePathType::AbsolutePathReference(vec![b"k".to_vec()]), + 99, + Some(vec![1, 2]), + ); + + assert_eq!(element.get_flags(), &Some(vec![1, 2])); + + { + let flags_mut = element.get_flags_mut(); + *flags_mut = Some(vec![9, 9]); + } + assert_eq!(element.get_flags(), &Some(vec![9, 9])); + + element.set_flags(None); + assert_eq!(element.get_flags(), &None); + + let owned = element.clone().get_flags_owned(); + assert_eq!(owned, None); +} + +#[test] +fn reference_with_sum_item_round_trips_through_bincode() { + let grove_version = GroveVersion::latest(); + let original = Element::new_reference_with_sum_item_with_max_hops_and_flags( + ReferencePathType::AbsolutePathReference(vec![b"a".to_vec(), b"bb".to_vec()]), + Some(5), + -42, + Some(vec![1, 2, 3]), + ); + let bytes = original.serialize(grove_version).expect("serialize ok"); + // Discriminant byte is pinned to 18. + assert_eq!(bytes[0], 18, "first byte must be discriminant 18"); + let back = Element::deserialize(&bytes, grove_version).expect("deserialize ok"); + assert_eq!(back, original); + + // Round trip through the NonCounted wrapper. + let wrapped = Element::new_non_counted(original).expect("wrap ok"); + let wrapped_bytes = wrapped.serialize(grove_version).expect("serialize ok"); + // First byte is the wrapper byte (15), second is the inner discriminant (18). + assert_eq!(wrapped_bytes[0], 15); + assert_eq!(wrapped_bytes[1], 18); + let back = Element::deserialize(&wrapped_bytes, grove_version).expect("deserialize ok"); + assert_eq!(back, wrapped); +} diff --git a/grovedb-element/tests/element_display_and_serialization.rs b/grovedb-element/tests/element_display_and_serialization.rs index 913c394a3..5d988ae83 100644 --- a/grovedb-element/tests/element_display_and_serialization.rs +++ b/grovedb-element/tests/element_display_and_serialization.rs @@ -98,6 +98,17 @@ fn element_display_and_type_helpers_cover_all_variants() { "dense_tree", "DenseAppendOnlyFixedSizeTree(count: 17, height: 18, flags: [19])", ), + ( + Element::ReferenceWithSumItem( + grovedb_element::reference_path::ReferencePathType::SiblingReference(b"k".to_vec()), + Some(4), + 42, + Some(vec![21]), + ), + ElementType::ReferenceWithSumItem, + "reference with sum item", + "ReferenceWithSumItem(SiblingReference(6b), max_hop: 4, sum: 42, flags: [21])", + ), ]; for (element, expected_type, expected_type_str, expected_display) in values { @@ -151,6 +162,15 @@ fn serialize_deserialize_round_trip_all_element_types_and_errors() { Element::new_mmr_tree(13, Some(vec![10])), Element::new_bulk_append_tree(14, 6, Some(vec![9])), Element::new_dense_tree(15, 7, Some(vec![8])), + Element::new_reference_with_sum_item_with_max_hops_and_flags( + grovedb_element::reference_path::ReferencePathType::AbsolutePathReference(vec![ + b"a".to_vec(), + b"b".to_vec(), + ]), + Some(4), + -42, + Some(vec![7]), + ), ]; for element in elements { @@ -236,6 +256,15 @@ fn element_display_without_flags_covers_none_branches() { Element::DenseAppendOnlyFixedSizeTree(17, 18, None), "DenseAppendOnlyFixedSizeTree(count: 17, height: 18)", ), + ( + Element::ReferenceWithSumItem( + ReferencePathType::SiblingReference(b"k".to_vec()), + None, + 7, + None, + ), + "ReferenceWithSumItem(SiblingReference(6b), max_hop: None, sum: 7)", + ), ]; for (element, expected_display) in values { diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index ef9eef543..edbfd8076 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -27,7 +27,8 @@ use crate::Element; #[cfg(feature = "minimal")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveOp, KeyInfoPath, TreeCache, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveOp, KeyInfoPath, + RefreshReferenceMode, TreeCache, }, Error, GroveDb, }; @@ -65,12 +66,21 @@ impl GroveOp { GroveOp::InsertTreeWithRootHash { flags, aggregate_data, + non_counted, + not_summed, + not_counted_or_summed, .. } => GroveDb::average_case_merk_insert_tree( key, flags, aggregate_data.parent_tree_type(), in_tree_type, + // Account for the wrapper byte if the op rebuilds the + // tree as `NonCounted(...)`, `NotSummed(...)`, or + // `NotCountedOrSummed(...)`. They share the same +1 + // discriminant overhead and are mutually exclusive on + // the rebuilt element. + super::wrapper_overhead_for(*non_counted, *not_summed, *not_counted_or_summed), propagate_if_input(), grove_version, ), @@ -114,19 +124,55 @@ impl GroveOp { GroveOp::RefreshReference { reference_path_type, max_reference_hop, + mode, flags, + non_counted, .. - } => GroveDb::average_case_merk_replace_element( - key, - &Element::Reference( - reference_path_type.clone(), - *max_reference_hop, - flags.clone(), - ), - in_tree_type, - propagate_if_input(), - grove_version, - ), + } => { + // Build the element shape the apply path will write, + // so the cost estimator includes any wrapper byte. + // Untrusted modes that preserve the on-disk sum use + // 0 as the cost-only stand-in (actual on-disk sum is + // not knowable here without a disk read; the wrapper + // byte is what matters for the estimate). + let inner = match mode { + RefreshReferenceMode::PlainReferenceTrusted + | RefreshReferenceMode::PlainReferenceUntrusted => Element::Reference( + reference_path_type.clone(), + *max_reference_hop, + flags.clone(), + ), + RefreshReferenceMode::SumItemReferenceTrusted(sum) + | RefreshReferenceMode::SumItemReferenceUntrustedValueUpdate(sum) => { + Element::ReferenceWithSumItem( + reference_path_type.clone(), + *max_reference_hop, + *sum, + flags.clone(), + ) + } + RefreshReferenceMode::SumItemReferenceUntrustedNoValueUpdate => { + Element::ReferenceWithSumItem( + reference_path_type.clone(), + *max_reference_hop, + 0, + flags.clone(), + ) + } + }; + let element = if *non_counted { + Element::NonCounted(Box::new(inner)) + } else { + inner + }; + GroveDb::average_case_merk_replace_element( + key, + &element, + in_tree_type, + propagate_if_input(), + grove_version, + ) + } GroveOp::Replace { element } => GroveDb::average_case_merk_replace_element( key, element, @@ -297,16 +343,23 @@ impl GroveOp { grove_version, ) } - GroveOp::InsertNonMerkTree { flags, meta, .. } => { - GroveDb::average_case_merk_insert_tree( - key, - flags, - meta.to_tree_type(), - in_tree_type, - propagate_if_input(), - grove_version, - ) - } + GroveOp::InsertNonMerkTree { + flags, + meta, + non_counted, + .. + } => GroveDb::average_case_merk_insert_tree( + key, + flags, + meta.to_tree_type(), + in_tree_type, + // `InsertNonMerkTree` only carries `non_counted` (the + // four non-Merk tree types are never sum-bearing, so + // `NotSummed` / `NotCountedOrSummed` can't apply). + if *non_counted { 1 } else { 0 }, + propagate_if_input(), + grove_version, + ), } } } @@ -1071,6 +1124,7 @@ mod tests { ReferencePathType::AbsolutePathReference(vec![b"target".to_vec()]), Some(5), None, + /* non_counted = */ false, true, )]; let mut paths = HashMap::new(); @@ -1117,6 +1171,150 @@ mod tests { ); } + #[test] + fn test_refresh_reference_with_sum_item_average_case_cost() { + let grove_version = GroveVersion::latest(); + let ops = vec![QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![vec![7]], + b"ref_key".to_vec(), + ReferencePathType::AbsolutePathReference(vec![b"target".to_vec()]), + Some(5), + 42, // sum_value + None, // flags + false, // non_counted + true, // trust_refresh_reference + )]; + let mut paths = HashMap::new(); + paths.insert( + KeyInfoPath(vec![]), + EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLevel(1, false), + estimated_layer_sizes: AllSubtrees(1, NoSumTrees, None), + }, + ); + paths.insert( + KeyInfoPath::from_known_owned_path(vec![vec![7]]), + EstimatedLayerInformation { + tree_type: TreeType::SumTree, + estimated_layer_count: PotentiallyAtMaxElements, + estimated_layer_sizes: AllItems(32, 64, None), + }, + ); + let result = GroveDb::estimated_case_operations_for_batch( + AverageCaseCostsType(paths), + ops, + None, + |_cost, _old_flags, _new_flags| Ok(false), + |_flags, _removed_key_bytes, _removed_value_bytes| { + Ok((NoStorageRemoval, NoStorageRemoval)) + }, + grove_version, + ) + .cost_as_result() + .expect("expected average case costs for refresh reference with sum item"); + // Same dispatch as `RefreshReference` but the rebuilt element carries + // an extra i64 sum, so the byte estimate is ~8 bytes larger. + assert!( + result.seek_count > 0, + "expected seek_count > 0, got {}", + result.seek_count + ); + assert!( + result.hash_node_calls > 0, + "expected hash_node_calls > 0, got {}", + result.hash_node_calls + ); + } + + #[test] + fn test_refresh_reference_with_sum_item_non_counted_average_case_cost() { + // Coverage for the `non_counted = true` branch of the + // RefreshReferenceWithSumItem cost arm. The estimator must + // build the same NonCounted(...) shape that execution writes, + // otherwise the serialized value is undercounted by the + // wrapper byte. + let grove_version = GroveVersion::latest(); + let nc_ops = vec![QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![vec![7]], + b"ref_key".to_vec(), + ReferencePathType::AbsolutePathReference(vec![b"target".to_vec()]), + Some(5), + 42, + None, + /* non_counted = */ true, + /* trust_refresh_reference = */ true, + )]; + let bare_ops = vec![QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![vec![7]], + b"ref_key".to_vec(), + ReferencePathType::AbsolutePathReference(vec![b"target".to_vec()]), + Some(5), + 42, + None, + /* non_counted = */ false, + /* trust_refresh_reference = */ true, + )]; + let mut paths = HashMap::new(); + paths.insert( + KeyInfoPath(vec![]), + EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLevel(1, false), + estimated_layer_sizes: AllSubtrees(1, NoSumTrees, None), + }, + ); + paths.insert( + KeyInfoPath::from_known_owned_path(vec![vec![7]]), + EstimatedLayerInformation { + tree_type: TreeType::CountSumTree, + estimated_layer_count: PotentiallyAtMaxElements, + estimated_layer_sizes: AllItems(32, 64, None), + }, + ); + let nc_cost = GroveDb::estimated_case_operations_for_batch( + AverageCaseCostsType(paths.clone()), + nc_ops, + None, + |_cost, _old_flags, _new_flags| Ok(false), + |_flags, _removed_key_bytes, _removed_value_bytes| { + Ok((NoStorageRemoval, NoStorageRemoval)) + }, + grove_version, + ) + .cost_as_result() + .expect("expected average case costs for non-counted refresh"); + let bare_cost = GroveDb::estimated_case_operations_for_batch( + AverageCaseCostsType(paths), + bare_ops, + None, + |_cost, _old_flags, _new_flags| Ok(false), + |_flags, _removed_key_bytes, _removed_value_bytes| { + Ok((NoStorageRemoval, NoStorageRemoval)) + }, + grove_version, + ) + .cost_as_result() + .expect("expected average case costs for bare refresh"); + + // The NonCounted-wrapped variant has at least one extra byte on + // the wire (the wrapper discriminant), so its cost estimate + // must be strictly larger than the bare variant (at least the + // +1 wrapper-discriminant byte). Strict `>` makes the test + // load-bearing: an undercount regression that produced an + // identical (under-counted) estimate — the exact bug being + // pinned — would fail this assertion. + assert!( + nc_cost.storage_cost.added_bytes + nc_cost.storage_cost.replaced_bytes + > bare_cost.storage_cost.added_bytes + bare_cost.storage_cost.replaced_bytes, + "non_counted=true cost must be strictly greater than bare cost; nc={:?}, bare={:?}", + nc_cost, + bare_cost, + ); + assert!(nc_cost.seek_count > 0); + assert!(nc_cost.hash_node_calls > 0); + } + #[test] fn test_patch_average_case_cost() { let grove_version = GroveVersion::latest(); @@ -1514,4 +1712,99 @@ mod tests { cost.storage_cost.added_bytes ); } + + /// Covers the wrapper-byte accounting in + /// `GroveOp::InsertTreeWithRootHash::average_case_cost`. Each + /// wrapper bit (`non_counted` / `not_summed` / + /// `not_counted_or_summed`) prepends one bincode discriminant byte + /// to the rebuilt tree element; the cost estimator must include it + /// in `value_len`. Before the fix the arm dropped the wrapper byte + /// entirely. The three bits are mutually exclusive on any element + /// — we vary one at a time. + #[test] + fn test_insert_tree_with_root_hash_wrapper_bits_average_case_cost_direct() { + let grove_version = GroveVersion::latest(); + let key = KeyInfo::KnownKey(b"merk_key".to_vec()); + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: ApproximateElements(0), + estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), + }; + let cost_for = |non_counted: bool, not_summed: bool, not_counted_or_summed: bool| { + let op = GroveOp::InsertTreeWithRootHash { + hash: [0xAAu8; 32], + root_key: None, + flags: None, + aggregate_data: AggregateData::NoAggregateData, + non_counted, + not_summed, + not_counted_or_summed, + }; + op.average_case_cost(&key, &layer_info, false, grove_version) + .cost_as_result() + .expect("expected cost for InsertTreeWithRootHash") + }; + let bare = cost_for(false, false, false); + let nc = cost_for(true, false, false); + let ns = cost_for(false, true, false); + let ncs = cost_for(false, false, true); + // Each wrapper bit must increase the estimated value_len by at + // least the +1 wrapper byte. We use `>=` because `add_bytes` + // can also pick up the varint-required-space overhead at + // payload-size boundaries. + assert!( + nc.storage_cost.added_bytes > bare.storage_cost.added_bytes, + "non_counted should add wrapper byte; nc={:?}, bare={:?}", + nc, + bare, + ); + assert!( + ns.storage_cost.added_bytes > bare.storage_cost.added_bytes, + "not_summed should add wrapper byte; ns={:?}, bare={:?}", + ns, + bare, + ); + assert!( + ncs.storage_cost.added_bytes > bare.storage_cost.added_bytes, + "not_counted_or_summed should add wrapper byte; ncs={:?}, bare={:?}", + ncs, + bare, + ); + } + + /// Covers the wrapper-byte accounting in + /// `GroveOp::InsertNonMerkTree::average_case_cost`. Non-Merk trees + /// only carry the `non_counted` wrapper bit (they're never + /// sum-bearing). + #[test] + fn test_insert_non_merk_tree_non_counted_average_case_cost_direct() { + let grove_version = GroveVersion::latest(); + let key = KeyInfo::KnownKey(b"inmerk_key".to_vec()); + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: ApproximateElements(0), + estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), + }; + let cost_for = |non_counted: bool| { + let op = GroveOp::InsertNonMerkTree { + hash: [0xCDu8; 32], + root_key: None, + flags: None, + aggregate_data: AggregateData::NoAggregateData, + meta: NonMerkTreeMeta::MmrTree { mmr_size: 50 }, + non_counted, + }; + op.average_case_cost(&key, &layer_info, false, grove_version) + .cost_as_result() + .expect("expected cost for InsertNonMerkTree") + }; + let bare = cost_for(false); + let nc = cost_for(true); + assert!( + nc.storage_cost.added_bytes > bare.storage_cost.added_bytes, + "InsertNonMerkTree non_counted should add wrapper byte; nc={:?}, bare={:?}", + nc, + bare, + ); + } } diff --git a/grovedb/src/batch/estimated_costs/mod.rs b/grovedb/src/batch/estimated_costs/mod.rs index 37b909da3..2ce500c20 100644 --- a/grovedb/src/batch/estimated_costs/mod.rs +++ b/grovedb/src/batch/estimated_costs/mod.rs @@ -16,6 +16,26 @@ pub mod average_case_costs; #[cfg(feature = "minimal")] pub mod worst_case_costs; +/// Cost-overhead in serialized bytes when a tree element will be +/// rebuilt wrapped in `Element::NonCounted`, `Element::NotSummed`, or +/// `Element::NotCountedOrSummed`. Each wrapper prepends one +/// discriminant byte to the on-disk payload. The three wrappers are +/// mutually exclusive on any element, so at most one of the input +/// flags is ever true. +#[cfg(feature = "minimal")] +#[inline] +pub(in crate::batch) fn wrapper_overhead_for( + non_counted: bool, + not_summed: bool, + not_counted_or_summed: bool, +) -> u32 { + if non_counted || not_summed || not_counted_or_summed { + 1 + } else { + 0 + } +} + /// Estimated costs types #[cfg(feature = "minimal")] pub enum EstimatedCostsType { diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 44d2678a3..ef572b442 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -28,7 +28,8 @@ use crate::Element; #[cfg(feature = "minimal")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveOp, KeyInfoPath, TreeCache, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveOp, KeyInfoPath, + RefreshReferenceMode, TreeCache, }, Error, GroveDb, }; @@ -64,12 +65,17 @@ impl GroveOp { GroveOp::InsertTreeWithRootHash { flags, aggregate_data, + non_counted, + not_summed, + not_counted_or_summed, .. } => GroveDb::worst_case_merk_insert_tree( key, flags, aggregate_data.parent_tree_type(), in_parent_tree_type, + // See the comment in the corresponding average-case arm. + super::wrapper_overhead_for(*non_counted, *not_summed, *not_counted_or_summed), propagate_if_input(), grove_version, ), @@ -107,19 +113,52 @@ impl GroveOp { GroveOp::RefreshReference { reference_path_type, max_reference_hop, + mode, flags, + non_counted, .. - } => GroveDb::worst_case_merk_replace_element( - key, - &Element::Reference( - reference_path_type.clone(), - *max_reference_hop, - flags.clone(), - ), - in_parent_tree_type, - propagate_if_input(), - grove_version, - ), + } => { + // Build the element shape the apply path will write — + // see the corresponding comment in the average-case + // estimator. + let inner = match mode { + RefreshReferenceMode::PlainReferenceTrusted + | RefreshReferenceMode::PlainReferenceUntrusted => Element::Reference( + reference_path_type.clone(), + *max_reference_hop, + flags.clone(), + ), + RefreshReferenceMode::SumItemReferenceTrusted(sum) + | RefreshReferenceMode::SumItemReferenceUntrustedValueUpdate(sum) => { + Element::ReferenceWithSumItem( + reference_path_type.clone(), + *max_reference_hop, + *sum, + flags.clone(), + ) + } + RefreshReferenceMode::SumItemReferenceUntrustedNoValueUpdate => { + Element::ReferenceWithSumItem( + reference_path_type.clone(), + *max_reference_hop, + 0, + flags.clone(), + ) + } + }; + let element = if *non_counted { + Element::NonCounted(Box::new(inner)) + } else { + inner + }; + GroveDb::worst_case_merk_replace_element( + key, + &element, + in_parent_tree_type, + propagate_if_input(), + grove_version, + ) + } GroveOp::Replace { element } => GroveDb::worst_case_merk_replace_element( key, element, @@ -301,11 +340,19 @@ impl GroveOp { propagate, grove_version, ), - GroveOp::InsertNonMerkTree { flags, meta, .. } => GroveDb::worst_case_merk_insert_tree( + GroveOp::InsertNonMerkTree { + flags, + meta, + non_counted, + .. + } => GroveDb::worst_case_merk_insert_tree( key, flags, meta.to_tree_type(), in_parent_tree_type, + // Non-Merk trees are never sum-bearing, so only the + // NonCounted wrapper applies. + if *non_counted { 1 } else { 0 }, propagate_if_input(), grove_version, ), @@ -829,6 +876,7 @@ mod tests { ReferencePathType::AbsolutePathReference(vec![b"target".to_vec()]), Some(5), None, + /* non_counted = */ false, true, )]; let mut paths = HashMap::new(); @@ -853,6 +901,114 @@ mod tests { assert!(cost.hash_node_calls > 0); } + #[test] + fn test_refresh_reference_with_sum_item_worst_case_cost() { + let grove_version = GroveVersion::latest(); + let ops = vec![QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![vec![7]], + b"ref_key".to_vec(), + ReferencePathType::AbsolutePathReference(vec![b"target".to_vec()]), + Some(5), + 42, // sum_value + None, // flags + false, // non_counted + true, // trust_refresh_reference + )]; + let mut paths = HashMap::new(); + paths.insert(KeyInfoPath(vec![]), MaxElementsNumber(1)); + paths.insert( + KeyInfoPath::from_known_owned_path(vec![vec![7]]), + MaxElementsNumber(100), + ); + let cost = GroveDb::estimated_case_operations_for_batch( + WorstCaseCostsType(paths), + ops, + None, + |_cost, _old_flags, _new_flags| Ok(false), + |_flags, _removed_key_bytes, _removed_value_bytes| { + Ok((NoStorageRemoval, NoStorageRemoval)) + }, + grove_version, + ) + .cost_as_result() + .expect("expected worst case costs for refresh reference with sum item"); + assert!(cost.seek_count > 0); + assert!(cost.hash_node_calls > 0); + } + + #[test] + fn test_refresh_reference_with_sum_item_non_counted_worst_case_cost() { + // Symmetric to the average-case test: verify the non_counted=true + // variant's worst-case estimate is at least as large as the + // bare variant. Before the fix the estimator dropped the + // NonCounted wrapper byte from the cost model. + let grove_version = GroveVersion::latest(); + let nc_ops = vec![QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![vec![7]], + b"ref_key".to_vec(), + ReferencePathType::AbsolutePathReference(vec![b"target".to_vec()]), + Some(5), + 42, + None, + /* non_counted = */ true, + /* trust_refresh_reference = */ true, + )]; + let bare_ops = vec![QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![vec![7]], + b"ref_key".to_vec(), + ReferencePathType::AbsolutePathReference(vec![b"target".to_vec()]), + Some(5), + 42, + None, + /* non_counted = */ false, + /* trust_refresh_reference = */ true, + )]; + let mut paths = HashMap::new(); + paths.insert(KeyInfoPath(vec![]), MaxElementsNumber(1)); + paths.insert( + KeyInfoPath::from_known_owned_path(vec![vec![7]]), + MaxElementsNumber(100), + ); + let nc_cost = GroveDb::estimated_case_operations_for_batch( + WorstCaseCostsType(paths.clone()), + nc_ops, + None, + |_cost, _old_flags, _new_flags| Ok(false), + |_flags, _removed_key_bytes, _removed_value_bytes| { + Ok((NoStorageRemoval, NoStorageRemoval)) + }, + grove_version, + ) + .cost_as_result() + .expect("expected worst case costs for non-counted refresh"); + let bare_cost = GroveDb::estimated_case_operations_for_batch( + WorstCaseCostsType(paths), + bare_ops, + None, + |_cost, _old_flags, _new_flags| Ok(false), + |_flags, _removed_key_bytes, _removed_value_bytes| { + Ok((NoStorageRemoval, NoStorageRemoval)) + }, + grove_version, + ) + .cost_as_result() + .expect("expected worst case costs for bare refresh"); + + // Strict `>`: NonCounted-wrapped element must be at least one + // wrapper-discriminant byte larger than the bare variant. The + // bug we're pinning is an undercount that produces an + // *identical* estimate, so equality must fail this check. + assert!( + nc_cost.storage_cost.added_bytes + nc_cost.storage_cost.replaced_bytes + > bare_cost.storage_cost.added_bytes + bare_cost.storage_cost.replaced_bytes, + "non_counted=true cost must be strictly greater than bare cost; nc={:?}, bare={:?}", + nc_cost, + bare_cost, + ); + assert!(nc_cost.seek_count > 0); + assert!(nc_cost.hash_node_calls > 0); + } + #[test] fn test_patch_worst_case_cost() { let grove_version = GroveVersion::latest(); @@ -1166,6 +1322,101 @@ mod tests { assert!(cost.hash_node_calls > 0); } + /// Covers the wrapper-byte accounting in + /// `GroveOp::InsertTreeWithRootHash::worst_case_cost`. The three + /// wrapper bits (`non_counted` / `not_summed` / + /// `not_counted_or_summed`) each prepend one bincode discriminant + /// byte to the rebuilt tree element; the cost estimator must + /// include it in `value_len`. Mirror of the average-case test. + #[test] + fn test_insert_tree_with_root_hash_wrapper_bits_worst_case_cost_direct() { + let grove_version = GroveVersion::latest(); + use grovedb_merk::tree::AggregateData; + let key = KeyInfo::KnownKey(b"merk_key".to_vec()); + let cost_for = |non_counted: bool, not_summed: bool, not_counted_or_summed: bool| { + let op = GroveOp::InsertTreeWithRootHash { + hash: [0xAAu8; 32], + root_key: None, + flags: None, + aggregate_data: AggregateData::NoAggregateData, + non_counted, + not_summed, + not_counted_or_summed, + }; + op.worst_case_cost( + &key, + TreeType::NormalTree, + &MaxElementsNumber(100), + false, + grove_version, + ) + .cost_as_result() + .expect("expected worst case cost for InsertTreeWithRootHash") + }; + let bare = cost_for(false, false, false); + let nc = cost_for(true, false, false); + let ns = cost_for(false, true, false); + let ncs = cost_for(false, false, true); + assert!( + nc.storage_cost.added_bytes > bare.storage_cost.added_bytes, + "non_counted should add wrapper byte; nc={:?}, bare={:?}", + nc, + bare, + ); + assert!( + ns.storage_cost.added_bytes > bare.storage_cost.added_bytes, + "not_summed should add wrapper byte; ns={:?}, bare={:?}", + ns, + bare, + ); + assert!( + ncs.storage_cost.added_bytes > bare.storage_cost.added_bytes, + "not_counted_or_summed should add wrapper byte; ncs={:?}, bare={:?}", + ncs, + bare, + ); + } + + /// Covers the wrapper-byte accounting in + /// `GroveOp::InsertNonMerkTree::worst_case_cost`. Non-Merk trees + /// only carry `non_counted`. + #[test] + fn test_insert_non_merk_tree_non_counted_worst_case_cost_direct() { + let grove_version = GroveVersion::latest(); + use grovedb_merk::tree::AggregateData; + let key = KeyInfo::KnownKey(b"new_dense".to_vec()); + let cost_for = |non_counted: bool| { + let op = GroveOp::InsertNonMerkTree { + hash: [5u8; 32], + root_key: None, + flags: None, + aggregate_data: AggregateData::NoAggregateData, + meta: NonMerkTreeMeta::DenseTree { + count: 0, + height: 8, + }, + non_counted, + }; + op.worst_case_cost( + &key, + TreeType::NormalTree, + &MaxElementsNumber(100), + false, + grove_version, + ) + .cost_as_result() + .expect("expected worst case cost for InsertNonMerkTree") + }; + let bare = cost_for(false); + let nc = cost_for(true); + assert!( + nc.storage_cost.added_bytes > bare.storage_cost.added_bytes, + "InsertNonMerkTree non_counted should add wrapper byte; nc={:?}, bare={:?}", + nc, + bare, + ); + } + #[test] fn test_replace_worst_case_cost() { let grove_version = GroveVersion::latest(); diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index db7215239..4834da99a 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -16,6 +16,7 @@ mod just_in_time_cost_tests; /// Just-in-time reference update handling for batch operations. pub mod just_in_time_reference_update; mod options; +mod refresh_reference_mode; #[cfg(test)] mod single_deletion_cost_tests; #[cfg(test)] @@ -72,13 +73,14 @@ use integer_encoding::VarInt; use itertools::Itertools; use key_info::{KeyInfo, KeyInfo::KnownKey}; pub use options::BatchApplyOptions; +pub use refresh_reference_mode::RefreshReferenceMode; pub use crate::batch::batch_structure::{OpsByLevelPath, OpsByPath}; #[cfg(feature = "estimated_costs")] use crate::batch::estimated_costs::EstimatedCostsType; use crate::{ batch::{batch_structure::BatchStructure, mode::BatchRunMode}, - element::MaxReferenceHop, + element::{MaxReferenceHop, SumValue}, operations::{delete::DeleteOptions, get::MAX_REFERENCE_HOPS, proof::util::hex_to_ascii}, reference_path::{ path_from_reference_path_type, path_from_reference_qualified_path_type, ReferencePathType, @@ -335,21 +337,51 @@ pub enum GroveOp { /// the wrapper byte on disk. non_counted: bool, }, - /// Refresh the reference with information provided - /// Providing this information is necessary to be able to calculate - /// average case and worst case costs - /// If TrustRefreshReference is true, then we do not query the element on - /// disk before write If it is false, the provided information is only - /// used for average case and worse case costs + /// Refresh a reference. The full op shape (which on-disk variant, + /// trust mode, sum-update behavior) lives in `mode` — see + /// [`RefreshReferenceMode`] for the per-variant contract. + /// + /// `non_counted` declares whether the rebuilt element is wrapped + /// in `NonCounted` (suppresses the count contribution in a + /// count-bearing parent). Under trusted variants it is written at + /// face value; under untrusted variants it is cross-checked + /// against on-disk and a mismatch is rejected (a silent wrapper + /// drop would corrupt the parent's count aggregate). + /// + /// Under trusted variants, the apply path writes the op's payload + /// (`reference_path_type`, `max_reference_hop`, `flags`, and the + /// mode's contained sum if any) verbatim. If on-disk has a + /// different variant or wrapper, it gets silently coerced — the + /// parent's count/sum aggregate may become inconsistent, caller's + /// responsibility. + /// + /// Under untrusted variants, the apply path reads on-disk, + /// cross-checks variant and wrapper, and writes back with the + /// on-disk path / max-hop / flags / wrapper. For + /// `SumItemReferenceUntrustedValueUpdate(v)` the op's `v` + /// overrides the on-disk sum; the other untrusted variants write + /// the on-disk element back verbatim. Op fields + /// `reference_path_type`, `max_reference_hop`, and `flags` are + /// used only for the average / worst case cost models in + /// untrusted mode. RefreshReference { - /// The type of reference path to use. + /// The reference path written under trusted variants. Under + /// untrusted variants the on-disk path is preserved; this + /// field is consulted only for the cost estimate. reference_path_type: ReferencePathType, - /// Maximum number of hops allowed when resolving the reference. + /// Max hops written under trusted variants. Same trust-mode + /// semantics as `reference_path_type`. max_reference_hop: MaxReferenceHop, - /// Optional element flags for the reference. + /// Fully specifies the op: on-disk variant, trust mode, and + /// sum-update behavior. See [`RefreshReferenceMode`]. + mode: RefreshReferenceMode, + /// Element flags written under trusted variants. Same + /// trust-mode semantics as `reference_path_type`. flags: Option, - /// If true, skip verifying the element on disk before writing. - trust_refresh_reference: bool, + /// Declares whether the rebuilt element is wrapped in + /// `NonCounted`. Trusted variants write at face value; + /// untrusted cross-check against on-disk. + non_counted: bool, }, /// Delete Delete, @@ -382,7 +414,12 @@ pub enum GroveOp { } impl GroveOp { - fn to_u8(&self) -> u8 { + /// Stable per-variant sort tag used by [`Ord::cmp`] and exposed + /// `pub(crate)` so tests can pin the exact value (not just relative + /// ordering). Changing any of these numbers is observable to + /// downstream sort-order assumptions in the batch pipeline; the + /// associated tests are intentionally strict. + pub(crate) fn to_u8(&self) -> u8 { match self { GroveOp::DeleteTree(..) => 0, // 1 used to be used for the DeleteSumTree @@ -640,12 +677,33 @@ impl fmt::Debug for QualifiedGroveDbOp { GroveOp::RefreshReference { reference_path_type, max_reference_hop, - trust_refresh_reference, + mode, + non_counted, .. } => { + let (label, mode_render) = match mode { + RefreshReferenceMode::PlainReferenceTrusted => { + ("Refresh Reference", "PlainReferenceTrusted".to_string()) + } + RefreshReferenceMode::PlainReferenceUntrusted => { + ("Refresh Reference", "PlainReferenceUntrusted".to_string()) + } + RefreshReferenceMode::SumItemReferenceTrusted(sum) => ( + "Refresh Reference With Sum Item", + format!("SumItemReferenceTrusted({sum})"), + ), + RefreshReferenceMode::SumItemReferenceUntrustedValueUpdate(sum) => ( + "Refresh Reference With Sum Item", + format!("SumItemReferenceUntrustedValueUpdate({sum})"), + ), + RefreshReferenceMode::SumItemReferenceUntrustedNoValueUpdate => ( + "Refresh Reference With Sum Item", + "SumItemReferenceUntrustedNoValueUpdate".to_string(), + ), + }; format!( - "Refresh Reference: path {:?}, max_hop {:?}, trust_reference {} ", - reference_path_type, max_reference_hop, trust_refresh_reference + "{label}: path {:?}, max_hop {:?}, mode {}, non_counted {} ", + reference_path_type, max_reference_hop, mode_render, non_counted, ) } GroveOp::Delete => "Delete".to_string(), @@ -810,14 +868,120 @@ impl QualifiedGroveDbOp { } } - /// A refresh reference op using a known owned path and known key + /// Construct a [`GroveOp::RefreshReference`] op for a plain + /// [`Element::Reference`] (no carried sum-item). Thin wrapper + /// that builds the unified `GroveOp::RefreshReference` with + /// `mode = PlainReferenceTrusted` or `PlainReferenceUntrusted` + /// based on `trust_refresh_reference`. + /// + /// `non_counted` declares whether the rebuilt element is wrapped + /// in `Element::NonCounted` (suppresses the count contribution + /// in a count-bearing parent). Under trusted mode it's written + /// at face value; under untrusted mode it's cross-checked + /// against the on-disk wrapper and a mismatch is rejected. + /// + /// See the [`RefreshReferenceMode`] doc for the trust-mode + /// contract. + /// + /// For sum-item-carrying references, use + /// [`Self::refresh_reference_with_sum_item_op`] (override the + /// sum) or [`Self::refresh_reference_with_sum_item_keep_sum_op`] + /// (preserve the on-disk sum). pub fn refresh_reference_op( path: Vec>, key: Vec, reference_path_type: ReferencePathType, max_reference_hop: MaxReferenceHop, flags: Option, + non_counted: bool, + trust_refresh_reference: bool, + ) -> Self { + let mode = if trust_refresh_reference { + RefreshReferenceMode::PlainReferenceTrusted + } else { + RefreshReferenceMode::PlainReferenceUntrusted + }; + let path = KeyInfoPath::from_known_owned_path(path); + Self { + path, + key: Some(KnownKey(key)), + op: GroveOp::RefreshReference { + reference_path_type, + max_reference_hop, + mode, + flags, + non_counted, + }, + } + } + + /// Construct a [`GroveOp::RefreshReference`] op for an + /// [`Element::ReferenceWithSumItem`] that **overrides** the + /// carried sum with the given `sum_value`. Thin wrapper that + /// builds the unified `GroveOp::RefreshReference` with `mode = + /// SumItemReferenceTrusted(sum_value)` or + /// `SumItemReferenceUntrustedValueUpdate(sum_value)` based on + /// `trust_refresh_reference`. + /// + /// See the [`RefreshReferenceMode`] doc for the trust-mode + /// contract. + /// + /// To refresh a `ReferenceWithSumItem`'s `value_hash` *without* + /// changing its carried sum, use + /// [`Self::refresh_reference_with_sum_item_keep_sum_op`]. + pub fn refresh_reference_with_sum_item_op( + path: Vec>, + key: Vec, + reference_path_type: ReferencePathType, + max_reference_hop: MaxReferenceHop, + sum_value: SumValue, + flags: Option, + non_counted: bool, trust_refresh_reference: bool, + ) -> Self { + let mode = if trust_refresh_reference { + RefreshReferenceMode::SumItemReferenceTrusted(sum_value) + } else { + RefreshReferenceMode::SumItemReferenceUntrustedValueUpdate(sum_value) + }; + let path = KeyInfoPath::from_known_owned_path(path); + Self { + path, + key: Some(KnownKey(key)), + op: GroveOp::RefreshReference { + reference_path_type, + max_reference_hop, + mode, + flags, + non_counted, + }, + } + } + + /// Construct a [`GroveOp::RefreshReference`] op for an + /// [`Element::ReferenceWithSumItem`] that **preserves** the + /// on-disk carried sum (no value update). Thin wrapper that + /// builds the unified `GroveOp::RefreshReference` with `mode = + /// SumItemReferenceUntrustedNoValueUpdate`. + /// + /// This op is **always untrusted** — under trusted mode the + /// apply path would have no sum to write without reading disk, + /// so the type system makes that combination unrepresentable + /// (there is no `SumItemReferenceTrustedNoValueUpdate` variant). + /// The on-disk element must be a `ReferenceWithSumItem` + /// (verified at apply); a plain `Reference` or any other variant + /// is rejected. + /// + /// Use this for the "I want to refresh my value_hash, leaving + /// the carried sum alone" case — caller doesn't need to know the + /// current sum. + pub fn refresh_reference_with_sum_item_keep_sum_op( + path: Vec>, + key: Vec, + reference_path_type: ReferencePathType, + max_reference_hop: MaxReferenceHop, + flags: Option, + non_counted: bool, ) -> Self { let path = KeyInfoPath::from_known_owned_path(path); Self { @@ -826,8 +990,9 @@ impl QualifiedGroveDbOp { op: GroveOp::RefreshReference { reference_path_type, max_reference_hop, + mode: RefreshReferenceMode::SumItemReferenceUntrustedNoValueUpdate, flags, - trust_refresh_reference, + non_counted, }, } } @@ -1224,19 +1389,33 @@ where .split_last() .expect("path validated non-empty above"); - let merk = match self.merks.entry(reference_path.to_vec()) { - HashMapEntry::Occupied(o) => o.into_mut(), - HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( - &mut cost, - (self.get_merk_fn)(reference_path, false) - )), - }; - - // Here the element being referenced doesn't change in the same batch - // and the max hop count is 1, meaning it should point directly to the base - // element at this point we can extract the value hash from the - // reference element directly + // Fast path: `recursions_allowed == 1` means the user-declared + // `max_reference_hop` budget allows exactly one more hop. Under + // the well-formed-user contract, that one hop must land on an + // `Item` (or `SumItem` / `ItemWithSumItem`) terminal — pointing + // at another `Reference` would violate the user's own budget. + // + // For an `Item` terminal the merk-stored `value_hash` IS the + // terminal's simple hash `H(serialize(item))`, which is exactly + // what `insert_reference` bakes into the dependent ref via + // `Op::PutCombinedReference`. So we can skip a full element + // decode and read the value_hash directly. + // + // Ill-formed input (`max_hop = 1` pointing at a `Reference`) + // is out of scope: this fast path would return the target's + // merk-combined hash as if it were a simple hash, producing a + // hash mismatch that `verify_grovedb` later reports. The + // contract is the user's to uphold; we don't pay the price of + // an extra dispatch on every well-formed hop=1 ref. if recursions_allowed == 1 { + let merk = match self.merks.entry(reference_path.to_vec()) { + HashMapEntry::Occupied(o) => o.into_mut(), + HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( + &mut cost, + (self.get_merk_fn)(reference_path, false) + )), + }; + let referenced_element_value_hash_opt = cost_return_on_error!( &mut cost, merk.get_value_hash( @@ -1266,8 +1445,16 @@ where .wrap_with_cost(OperationCost::default()) ); - Ok(referenced_element_value_hash).wrap_with_cost(cost) - } else if let Some(referenced_path) = intermediate_reference_info { + return Ok(referenced_element_value_hash).wrap_with_cost(cost); + } + + // Slow path: `recursions_allowed > 1`. Dispatch on whether the + // target is being modified in this same batch. Neither branch + // needs the merk handle here — the helpers open (or reuse the + // cached) merk themselves via `self.merks.entry(..)`. + if let Some(referenced_path) = intermediate_reference_info { + // Target is in batch (refresh). Hop through the op's new + // path; budget decrements by one for this hop. let path = cost_return_on_error_into_no_add!( cost, path_from_reference_qualified_path_type(referenced_path.clone(), qualified_path) @@ -1282,10 +1469,9 @@ where grove_version, ) } else { - // Here the element being referenced doesn't change in the same batch - // but the hop count is greater than 1, we can't just take the value hash from - // the referenced element as an element further down in the chain might still - // change in the batch. + // Target is not in batch. Read the on-disk element and + // dispatch by type (Item terminals return their simple + // hash; References recurse). self.process_reference_with_hop_count_greater_than_one( key, reference_path, @@ -1472,7 +1658,9 @@ where let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); Ok(val_hash).wrap_with_cost(cost) } - Element::Reference(path, ..) => { + // Both reference variants follow the same chain-resolution path + // to compute their effective value hash. + Element::Reference(path, ..) | Element::ReferenceWithSumItem(path, ..) => { let path = cost_return_on_error_into_no_add!( cost, path_from_reference_qualified_path_type(path.clone(), qualified_path) @@ -1623,7 +1811,8 @@ where } } } - Element::Reference(path, ..) => { + // Both reference variants follow the same chain. + Element::Reference(path, ..) | Element::ReferenceWithSumItem(path, ..) => { let path = cost_return_on_error_into_no_add!( cost, path_from_reference_qualified_path_type( @@ -1675,7 +1864,7 @@ where let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); Ok(val_hash).wrap_with_cost(cost) } - Element::Reference(path, ..) => { + Element::Reference(path, ..) | Element::ReferenceWithSumItem(path, ..) => { let path = cost_return_on_error_into_no_add!( cost, path_from_reference_qualified_path_type(path.clone(), qualified_path) @@ -1715,11 +1904,26 @@ where }, GroveOp::RefreshReference { reference_path_type, - trust_refresh_reference, + mode, .. } => { - // We are pointing towards a reference that will be refreshed - let reference_info = if *trust_refresh_reference { + // We are pointing towards a reference that will + // be refreshed in this batch. The dependent + // ref's value hash must be computed against + // whatever the apply path will write — which + // depends on the trust mode encoded in `mode`: + // + // * Trusted variants: apply writes the op's + // payload (`reference_path_type`). Thread it + // through so dependent refs resolve against + // the post-batch path. + // + // * Untrusted variants: apply keeps the on-disk + // path (sum-item updates only override the + // carried sum; the path is preserved). Pass + // `None` so `process_reference` resolves + // through the (unchanged) on-disk path. + let reference_info = if mode.is_trusted() { Some(reference_path_type) } else { None @@ -1858,8 +2062,12 @@ where }; // Check tree-override protection for all non-reference elements. + // `is_reference()` looks through `NonCounted` and recognizes + // both `Element::Reference` and `Element::ReferenceWithSumItem`, + // so wrapped or sum-bearing references receive the same + // exemption as plain references. if batch_apply_options.validate_insertion_does_not_override_tree - && !matches!(&element, Element::Reference(..)) + && !element.is_reference() { let merk = self.merks.get_mut(path).expect("the Merk is cached"); let maybe_existing = cost_return_on_error_into!( @@ -1927,7 +2135,21 @@ where // element_at_key_already_exists) are wrapper-aware via the // helper methods updated in grovedb-element. match element.underlying() { - Element::Reference(path_reference, element_max_reference_hop, _) => { + // Both reference variants share this batch-insert + // path. `ReferenceWithSumItem` has a 4-tuple shape + // (path, max_hop, sum_value, flags); we only bind + // the path and the max-hop here — the sum value is + // included in the element's serialized bytes via + // `insert_reference_into_batch_operations` and is + // picked up by `get_feature_type` for the parent's + // sum aggregation. + Element::Reference(path_reference, element_max_reference_hop, _) + | Element::ReferenceWithSumItem( + path_reference, + element_max_reference_hop, + _, + _, + ) => { // Check existence for InsertIfNotExists on references if is_insert_if_not_exists || batch_apply_options.validate_insertion_does_not_override @@ -2127,58 +2349,189 @@ where GroveOp::RefreshReference { reference_path_type, max_reference_hop, + mode, flags, - trust_refresh_reference, + non_counted, } => { - // We have a refresh reference Op, this means we need to get the actual - // reference element on disk first - - let element = if trust_refresh_reference { - Element::Reference(reference_path_type, max_reference_hop, flags) - } else { - let merk = self.merks.get(path).expect("the Merk is cached"); - let value = cost_return_on_error!( - &mut cost, - merk.get( - key_info.as_slice(), - true, - Some(Element::value_defined_cost_for_serialized_value), - grove_version - ) - .map( - |result_value| result_value.map_err(Error::MerkError).and_then( - |maybe_value| maybe_value.ok_or(Error::InvalidInput( - "trying to refresh a non existing reference", + // Five-way dispatch on the mode variant. Trust + // mode is encoded in the variant name — see + // `RefreshReferenceMode` for the per-variant + // contract. + let wrap_if_non_counted = |inner: Element| -> Result { + if non_counted { + Element::new_non_counted(inner).map_err(|e| { + Error::CorruptedData(format!( + "failed to wrap refreshed reference in NonCounted: {e}" + )) + }) + } else { + Ok(inner) + } + }; + let element = match mode { + // ---------- Trusted variants ---------- + // Build the element from op fields verbatim, + // no disk read. If on-disk has a different + // variant or wrapper, it gets silently + // coerced — caller-asserted shape. + RefreshReferenceMode::PlainReferenceTrusted => { + let inner = + Element::Reference(reference_path_type, max_reference_hop, flags); + cost_return_on_error_no_add!(cost, wrap_if_non_counted(inner)) + } + RefreshReferenceMode::SumItemReferenceTrusted(sum) => { + let inner = Element::ReferenceWithSumItem( + reference_path_type, + max_reference_hop, + sum, + flags, + ); + cost_return_on_error_no_add!(cost, wrap_if_non_counted(inner)) + } + // ---------- Untrusted variants ---------- + // Read on-disk, cross-check variant + + // wrapper, then either write back verbatim + // or override the sum. + RefreshReferenceMode::PlainReferenceUntrusted + | RefreshReferenceMode::SumItemReferenceUntrustedValueUpdate(_) + | RefreshReferenceMode::SumItemReferenceUntrustedNoValueUpdate => { + let merk = self.merks.get(path).expect("the Merk is cached"); + let value = cost_return_on_error!( + &mut cost, + merk.get( + key_info.as_slice(), + true, + Some(Element::value_defined_cost_for_serialized_value), + grove_version + ) + .map(|result_value| result_value + .map_err(Error::MerkError) + .and_then(|maybe_value| maybe_value.ok_or( + Error::InvalidInput( + "trying to refresh a non existing reference", + ) + ))) + ); + let on_disk = cost_return_on_error_no_add!( + cost, + Element::deserialize(value.as_slice(), grove_version).map_err( + |e| { + Error::CorruptedData(format!( + "unable to deserialize element: {e}" + )) + } + ) + ); + if on_disk.is_non_counted() != non_counted { + return Err(Error::InvalidInput( + "RefreshReference non_counted flag disagrees with on-disk \ + wrapper", + )) + .wrap_with_cost(cost); + } + match (mode, on_disk.underlying()) { + ( + RefreshReferenceMode::PlainReferenceUntrusted, + Element::Reference(..), + ) + | ( + RefreshReferenceMode::SumItemReferenceUntrustedNoValueUpdate, + Element::ReferenceWithSumItem(..), + ) => on_disk, + ( + RefreshReferenceMode::SumItemReferenceUntrustedValueUpdate(sum), + Element::ReferenceWithSumItem( + disk_path, + disk_max_hop, + _disk_sum, + disk_flags, + ), + ) => { + let rebuilt_inner = Element::ReferenceWithSumItem( + disk_path.clone(), + *disk_max_hop, + sum, + disk_flags.clone(), + ); + cost_return_on_error_no_add!( + cost, + wrap_if_non_counted(rebuilt_inner) + ) + } + (RefreshReferenceMode::PlainReferenceUntrusted, _) => { + return Err(Error::InvalidInput( + "RefreshReference PlainReferenceUntrusted applied to \ + non-plain-Reference on disk", )) + .wrap_with_cost(cost); + } + ( + RefreshReferenceMode::SumItemReferenceUntrustedValueUpdate(_), + _, ) - ) - ); - cost_return_on_error_no_add!( - cost, - Element::deserialize(value.as_slice(), grove_version).map_err(|e| { - Error::CorruptedData(format!("unable to deserialize element: {e}")) - }) - ) + | ( + RefreshReferenceMode::SumItemReferenceUntrustedNoValueUpdate, + _, + ) => { + return Err(Error::InvalidInput( + "RefreshReference SumItem-untrusted mode applied to \ + non-RefWithSumItem on disk", + )) + .wrap_with_cost(cost); + } + // Trusted variants are handled in + // the outer match and never reach + // this point. + ( + RefreshReferenceMode::PlainReferenceTrusted + | RefreshReferenceMode::SumItemReferenceTrusted(_), + _, + ) => unreachable!("trusted modes handled in outer match"), + } + } }; - // Look through `NonCounted` so a wrapped reference can - // still be refreshed. The wrapper is transparent for - // refresh; only the inner reference's path matters. - let Element::Reference(path_reference, max_reference_hop, _) = - element.underlying() - else { - return Err(Error::InvalidInput( - "trying to refresh a an element that is not a reference", + // Mirror the per-merk wrapper invariant enforced + // for direct inserts: a NonCounted-wrapped + // element may only live in a count-bearing + // parent. Without this guard a trusted refresh + // with `non_counted=true` could persist a + // NonCounted wrapper into a non-count-bearing + // tree. + if element.is_non_counted() && !in_tree_type.is_count_bearing() { + return Err(Error::InvalidBatchOperation( + "RefreshReference with non_counted=true requires a count-bearing parent", )) .wrap_with_cost(cost); + } + + let (path_reference, max_reference_hop) = match element.underlying() { + Element::Reference(path, max_hop, _) => (path.clone(), *max_hop), + Element::ReferenceWithSumItem(path, max_hop, _, _) => { + (path.clone(), *max_hop) + } + _ => { + // Unreachable: branches above always + // produce one of these two variants + // (possibly NonCounted-wrapped). + return Err(Error::InvalidInput( + "internal: refresh did not produce a reference variant", + )) + .wrap_with_cost(cost); + } }; - let merk_feature_type = in_tree_type.empty_tree_feature_type(); + let merk_feature_type = cost_return_on_error_into!( + &mut cost, + element + .get_feature_type(in_tree_type) + .wrap_with_cost(OperationCost::default()) + ); let path_reference = cost_return_on_error_into!( &mut cost, path_from_reference_path_type( - path_reference.clone(), + path_reference, path, Some(key_info.as_slice()) ) diff --git a/grovedb/src/batch/refresh_reference_mode.rs b/grovedb/src/batch/refresh_reference_mode.rs new file mode 100644 index 000000000..db538998c --- /dev/null +++ b/grovedb/src/batch/refresh_reference_mode.rs @@ -0,0 +1,93 @@ +//! Mode discriminant for [`GroveOp::RefreshReference`]. +//! +//! Encodes both the on-disk shape being refreshed (plain `Reference` +//! vs. `ReferenceWithSumItem`) and the trust mode +//! (caller-asserted-shape vs. read-and-validate-on-disk) in a single +//! enum so invalid combinations are unrepresentable. +//! +//! [`GroveOp::RefreshReference`]: super::GroveOp::RefreshReference + +#[cfg(feature = "minimal")] +use crate::element::SumValue; + +/// Fully specifies a [`GroveOp::RefreshReference`] op: which on-disk +/// shape is being refreshed, the trust mode, and (for sum-item value +/// updates) the new carried sum. +/// +/// Trust mode is encoded in the variant itself, so invalid +/// combinations are unrepresentable. In particular, "refresh a +/// `ReferenceWithSumItem` without changing its carried sum" only +/// makes sense in untrusted mode (under trusted the apply path has +/// no sum to write without reading disk) — only the +/// `Untrusted...NoValueUpdate` variant covers that case; there is +/// no trusted counterpart. +/// +/// * [`Self::PlainReferenceTrusted`]: refresh a plain +/// [`Element::Reference`]; apply writes the op's payload +/// verbatim. No disk read; if on-disk is not a plain `Reference` +/// it gets silently coerced (caller asserts the shape). +/// +/// * [`Self::PlainReferenceUntrusted`]: refresh a plain +/// [`Element::Reference`]; apply reads on-disk and writes it +/// back. A non-plain-`Reference` on disk is rejected. +/// +/// * [`Self::SumItemReferenceTrusted`]: refresh an +/// [`Element::ReferenceWithSumItem`] with the contained +/// [`SumValue`]; apply writes the op's payload verbatim with that +/// sum. Cross-type coercion is the caller's responsibility. +/// +/// * [`Self::SumItemReferenceUntrustedValueUpdate`]: refresh an +/// [`Element::ReferenceWithSumItem`]; apply reads on-disk for +/// path/wrapper and overrides the carried sum with the contained +/// value. On-disk must be `ReferenceWithSumItem`. +/// +/// * [`Self::SumItemReferenceUntrustedNoValueUpdate`]: refresh an +/// [`Element::ReferenceWithSumItem`]; apply reads on-disk and +/// writes it back verbatim, preserving the carried sum. On-disk +/// must be `ReferenceWithSumItem`. +/// +/// [`Element::Reference`]: crate::Element::Reference +/// [`Element::ReferenceWithSumItem`]: crate::Element::ReferenceWithSumItem +/// [`GroveOp::RefreshReference`]: super::GroveOp::RefreshReference +#[cfg(feature = "minimal")] +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +pub enum RefreshReferenceMode { + /// Trusted refresh of a plain [`Element::Reference`]. + /// + /// [`Element::Reference`]: crate::Element::Reference + PlainReferenceTrusted, + /// Untrusted refresh of a plain [`Element::Reference`]. + /// + /// [`Element::Reference`]: crate::Element::Reference + PlainReferenceUntrusted, + /// Trusted refresh of an [`Element::ReferenceWithSumItem`] with + /// the contained sum value. + /// + /// [`Element::ReferenceWithSumItem`]: crate::Element::ReferenceWithSumItem + SumItemReferenceTrusted(SumValue), + /// Untrusted refresh of an [`Element::ReferenceWithSumItem`] + /// that overrides the carried sum with the contained value. + /// + /// [`Element::ReferenceWithSumItem`]: crate::Element::ReferenceWithSumItem + SumItemReferenceUntrustedValueUpdate(SumValue), + /// Untrusted refresh of an [`Element::ReferenceWithSumItem`] + /// that preserves the on-disk carried sum. + /// + /// [`Element::ReferenceWithSumItem`]: crate::Element::ReferenceWithSumItem + SumItemReferenceUntrustedNoValueUpdate, +} + +#[cfg(feature = "minimal")] +impl RefreshReferenceMode { + /// True for the variants where the apply path uses the op's + /// declared shape verbatim (no on-disk read for the writeable + /// fields). Used by `follow_reference_get_value_hash` to decide + /// whether dependent refs should resolve against the op's path + /// or the on-disk path. + pub fn is_trusted(&self) -> bool { + matches!( + self, + Self::PlainReferenceTrusted | Self::SumItemReferenceTrusted(_) + ) + } +} diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index d1204131a..38bfc0e0d 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -660,86 +660,89 @@ fn query_item_to_grovedb(item: QueryItem) -> crate::QueryItem { } } -fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { - match element { - crate::Element::Item(value, element_flags) => grovedbg_types::Element::Item { - value, - element_flags, - }, - crate::Element::Tree(root_key, element_flags) => grovedbg_types::Element::Subtree { - root_key, - element_flags, - }, - crate::Element::Reference( - ReferencePathType::AbsolutePathReference(path), - _, - element_flags, - ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::AbsolutePathReference { - path, - element_flags, - }), - crate::Element::Reference( - ReferencePathType::UpstreamRootHeightReference(n_keep, path_append), - _, - element_flags, - ) => grovedbg_types::Element::Reference( - grovedbg_types::Reference::UpstreamRootHeightReference { - n_keep: n_keep.into(), - path_append, +/// Convert a [`crate::ReferencePathType`] plus optional element flags +/// into the corresponding `grovedbg_types::Reference` wire variant. +/// Shared by both the plain `Element::Reference` and the +/// `Element::ReferenceWithSumItem` arms of [`element_to_grovedbg`]. +fn reference_path_to_grovedbg( + reference_path: ReferencePathType, + element_flags: Option>, +) -> grovedbg_types::Reference { + match reference_path { + ReferencePathType::AbsolutePathReference(path) => { + grovedbg_types::Reference::AbsolutePathReference { + path, element_flags, - }, - ), - crate::Element::Reference( - ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( - n_keep, - path_append, - ), - _, - element_flags, - ) => grovedbg_types::Element::Reference( - grovedbg_types::Reference::UpstreamRootHeightWithParentPathAdditionReference { + } + } + ReferencePathType::UpstreamRootHeightReference(n_keep, path_append) => { + grovedbg_types::Reference::UpstreamRootHeightReference { n_keep: n_keep.into(), path_append, element_flags, - }, - ), - crate::Element::Reference( - ReferencePathType::UpstreamFromElementHeightReference(n_remove, path_append), - _, + } + } + ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( + n_keep, + path_append, + ) => grovedbg_types::Reference::UpstreamRootHeightWithParentPathAdditionReference { + n_keep: n_keep.into(), + path_append, element_flags, - ) => grovedbg_types::Element::Reference( + }, + ReferencePathType::UpstreamFromElementHeightReference(n_remove, path_append) => { grovedbg_types::Reference::UpstreamFromElementHeightReference { n_remove: n_remove.into(), path_append, element_flags, - }, - ), - crate::Element::Reference( - ReferencePathType::CousinReference(swap_parent), - _, - element_flags, - ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::CousinReference { - swap_parent, - element_flags, - }), - crate::Element::Reference( - ReferencePathType::RemovedCousinReference(swap_parent), - _, - element_flags, - ) => { - grovedbg_types::Element::Reference(grovedbg_types::Reference::RemovedCousinReference { + } + } + ReferencePathType::CousinReference(swap_parent) => { + grovedbg_types::Reference::CousinReference { swap_parent, element_flags, - }) + } } - crate::Element::Reference( - ReferencePathType::SiblingReference(sibling_key), - _, + ReferencePathType::RemovedCousinReference(swap_parent) => { + grovedbg_types::Reference::RemovedCousinReference { + swap_parent, + element_flags, + } + } + ReferencePathType::SiblingReference(sibling_key) => { + grovedbg_types::Reference::SiblingReference { + sibling_key, + element_flags, + } + } + } +} + +fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { + match element { + crate::Element::Item(value, element_flags) => grovedbg_types::Element::Item { + value, element_flags, - ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::SiblingReference { - sibling_key, + }, + crate::Element::Tree(root_key, element_flags) => grovedbg_types::Element::Subtree { + root_key, element_flags, - }), + }, + crate::Element::Reference(reference_path, _, element_flags) => { + grovedbg_types::Element::Reference(reference_path_to_grovedbg( + reference_path, + element_flags, + )) + } + crate::Element::ReferenceWithSumItem( + reference_path, + _max_hop, + sum_item_value, + element_flags, + ) => grovedbg_types::Element::ReferenceWithSumItem { + reference: reference_path_to_grovedbg(reference_path, element_flags), + sum_item_value, + }, crate::Element::SumItem(value, element_flags) => grovedbg_types::Element::SumItem { value, element_flags, @@ -892,4 +895,69 @@ mod tests { _ => panic!("unexpected debugger conversion"), } } + + /// `ReferenceWithSumItem` is converted to the dedicated + /// `grovedbg_types::Element::ReferenceWithSumItem` wire variant + /// — the path is forwarded via the shared + /// `reference_path_to_grovedbg` helper and the explicit + /// `sum_item_value` (independent of the resolved target) is + /// preserved on the wire. + #[test] + fn element_to_grovedbg_converts_reference_with_sum_item_absolute_path() { + let flags = Some(vec![9, 9, 9]); + let path = ReferencePathType::AbsolutePathReference(vec![ + b"some_leaf".to_vec(), + b"target".to_vec(), + ]); + let element = crate::Element::ReferenceWithSumItem(path, Some(3), 42, flags.clone()); + match element_to_grovedbg(element) { + grovedbg_types::Element::ReferenceWithSumItem { + reference, + sum_item_value, + } => { + assert_eq!(sum_item_value, 42); + match reference { + grovedbg_types::Reference::AbsolutePathReference { + path, + element_flags, + } => { + assert_eq!(path, vec![b"some_leaf".to_vec(), b"target".to_vec()]); + assert_eq!(element_flags, flags); + } + other => panic!("unexpected wire reference: {other:?}"), + } + } + other => panic!("unexpected debugger conversion: {other:?}"), + } + } + + /// A non-absolute reference-with-sum-item (here `SiblingReference`) + /// also flows through `reference_path_to_grovedbg` unchanged. + /// Exercises one of the six non-absolute path variants to confirm + /// the helper covers the full discriminant set. + #[test] + fn element_to_grovedbg_converts_reference_with_sum_item_sibling() { + let element = crate::Element::ReferenceWithSumItem( + ReferencePathType::SiblingReference(b"sib".to_vec()), + None, + -7, + None, + ); + match element_to_grovedbg(element) { + grovedbg_types::Element::ReferenceWithSumItem { + reference, + sum_item_value, + } => { + assert_eq!(sum_item_value, -7); + assert!(matches!( + reference, + grovedbg_types::Reference::SiblingReference { + sibling_key, + element_flags: None, + } if sibling_key == b"sib".to_vec() + )); + } + other => panic!("unexpected debugger conversion: {other:?}"), + } + } } diff --git a/grovedb/src/element/aggregate_sum_query/mod.rs b/grovedb/src/element/aggregate_sum_query/mod.rs index 33f4ebc15..d4f753630 100644 --- a/grovedb/src/element/aggregate_sum_query/mod.rs +++ b/grovedb/src/element/aggregate_sum_query/mod.rs @@ -410,11 +410,19 @@ impl ElementAggregateSumQueryExtensions for Element { .convert_if_reference_to_absolute_reference(args.path, args.key); let element = cost_return_on_error_into_no_add!(cost, element); - let Element::Reference(ref_path, _, _) = element else { - return Err(Error::InternalError( - "expected a reference after conversion".to_string(), - )) - .wrap_with_cost(cost); + // Aggregate-sum follows the reference chain to a SumItem. + // `ReferenceWithSumItem` is also a reference and resolves the + // same way; its carried sum is a parent-aggregation property + // and does not affect the chain destination. + let ref_path = match element { + Element::Reference(ref_path, _, _) + | Element::ReferenceWithSumItem(ref_path, _, _, _) => ref_path, + _ => { + return Err(Error::InternalError( + "expected a reference after conversion".to_string(), + )) + .wrap_with_cost(cost); + } }; let mut current_qualified_path = match ref_path { @@ -463,7 +471,9 @@ impl ElementAggregateSumQueryExtensions for Element { ); match resolved { - Element::Reference(next_ref_path, _, _) => { + // Both reference variants continue the chain. + Element::Reference(next_ref_path, _, _) + | Element::ReferenceWithSumItem(next_ref_path, _, _, _) => { if hops_left == 0 { return Err(Error::ReferenceLimit).wrap_with_cost(cost); } diff --git a/grovedb/src/element/aggregate_sum_query/tests.rs b/grovedb/src/element/aggregate_sum_query/tests.rs index f8ca72d03..c7e128195 100644 --- a/grovedb/src/element/aggregate_sum_query/tests.rs +++ b/grovedb/src/element/aggregate_sum_query/tests.rs @@ -1738,6 +1738,125 @@ fn test_reference_to_item_with_sum_item_followed() { assert_eq!(result.results, vec![(b"ref_hybrid".to_vec(), 15)]); } +#[test] +fn test_reference_with_sum_item_chain_followed_to_sum_item() { + // A `ReferenceWithSumItem` is also a reference: aggregate-sum-query + // should follow the chain to a `SumItem` target. The sum value carried + // on the variant is a parent-aggregation property and does NOT affect + // what aggregate-sum-query returns (which is the target's sum). + let grove_version = GroveVersion::latest(); + let db = make_test_sum_tree_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"target", + Element::new_sum_item(7), + None, + None, + grove_version, + ) + .unwrap() + .expect("cannot insert sum item target"); + // Insert a ReferenceWithSumItem pointing to the sum item "target". + // The carried sum (999) is INDEPENDENT of the target's sum (7); the + // query returns the target's value, not the carried sum. + db.insert( + [TEST_LEAF].as_ref(), + b"link", + Element::new_reference_with_sum_item( + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]), + 999, + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("cannot insert ref-with-sum-item"); + + let aggregate_sum_query = AggregateSumQuery::new_single_key(b"link".to_vec(), 100); + let aggregate_sum_path_query = AggregateSumPathQuery { + path: vec![TEST_LEAF.to_vec()], + aggregate_sum_query, + }; + + let result = Element::get_aggregate_sum_query( + &db.db, + &aggregate_sum_path_query, + AggregateSumQueryOptions::default(), + None, + grove_version, + ) + .unwrap() + .expect("expected successful get_query"); + + assert_eq!(result.results, vec![(b"link".to_vec(), 7)]); +} + +#[test] +fn test_reference_with_sum_item_chain_through_intermediate_reference() { + // `ReferenceWithSumItem` -> `Reference` -> `SumItem`. Exercises the + // chain-continuation arm in `aggregate_sum_query::process_reference` + // for the new variant. + let grove_version = GroveVersion::latest(); + let db = make_test_sum_tree_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"target", + Element::new_sum_item(13), + None, + None, + grove_version, + ) + .unwrap() + .expect("cannot insert sum item target"); + db.insert( + [TEST_LEAF].as_ref(), + b"middle", + Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"target".to_vec(), + ])), + None, + None, + grove_version, + ) + .unwrap() + .expect("cannot insert middle reference"); + db.insert( + [TEST_LEAF].as_ref(), + b"head", + Element::new_reference_with_sum_item( + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"middle".to_vec()]), + 999, + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("cannot insert head ref-with-sum-item"); + + let aggregate_sum_query = AggregateSumQuery::new_single_key(b"head".to_vec(), 100); + let aggregate_sum_path_query = AggregateSumPathQuery { + path: vec![TEST_LEAF.to_vec()], + aggregate_sum_query, + }; + + let result = Element::get_aggregate_sum_query( + &db.db, + &aggregate_sum_path_query, + AggregateSumQueryOptions::default(), + None, + grove_version, + ) + .unwrap() + .expect("expected successful get_query"); + + assert_eq!(result.results, vec![(b"head".to_vec(), 13)]); +} + #[test] fn test_reference_to_regular_item_errors() { // A reference that resolves to a regular Item (not a sum item) should error diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 9982657b4..ff5d42cca 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -188,6 +188,7 @@ impl GroveDb { flags: &Option, tree_type: TreeType, in_parent_tree_type: TreeType, + wrapper_overhead: u32, propagate_if_input: Option<&EstimatedLayerInformation>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -207,7 +208,11 @@ impl GroveDb { flags_len + flags_len.required_space() as u32 }); let tree_cost_size = tree_type.cost_size(); - let value_len = tree_cost_size + flags_len; + // `wrapper_overhead` accounts for the wrapper discriminant byte + // that `Element::NonCounted` / `Element::NotSummed` / + // `Element::NotCountedOrSummed` add to the serialized payload. + // Callers pass 1 when the on-disk shape will be wrapped, else 0. + let value_len = tree_cost_size + flags_len + wrapper_overhead; add_cost_case_merk_insert_layered(&mut cost, key_len, value_len, in_parent_tree_type); if let Some(input) = propagate_if_input { add_average_case_merk_propagate(&mut cost, input, grove_version) diff --git a/grovedb/src/estimated_costs/worst_case_costs.rs b/grovedb/src/estimated_costs/worst_case_costs.rs index 5f189aa43..9aaa26c1e 100644 --- a/grovedb/src/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/estimated_costs/worst_case_costs.rs @@ -105,6 +105,7 @@ impl GroveDb { flags: &Option, tree_type: TreeType, in_parent_tree_type: TreeType, + wrapper_overhead: u32, propagate_if_input: Option<&WorstCaseLayerInformation>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -124,7 +125,9 @@ impl GroveDb { flags_len + flags_len.required_space() as u32 }); let tree_cost = tree_type.cost_size(); - let value_len = tree_cost + flags_len; + // `wrapper_overhead` accounts for the wrapper discriminant byte + // (see corresponding comment in `average_case_merk_insert_tree`). + let value_len = tree_cost + flags_len + wrapper_overhead; add_cost_case_merk_insert_layered(&mut cost, key_len, value_len, in_parent_tree_type); if let Some(input) = propagate_if_input { add_worst_case_merk_propagate(&mut cost, input).map_err(Error::MerkError) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 911b07677..cfc8a8936 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -1059,8 +1059,12 @@ impl GroveDb { ); } } - Element::Reference(ref reference_path, ..) => { - // Skip this whole check if we don't `verify_references` + Element::Reference(ref reference_path, ..) + | Element::ReferenceWithSumItem(ref reference_path, ..) => { + // Skip this whole check if we don't `verify_references`. + // `ReferenceWithSumItem` shares this verification path — + // the sum is hashed as part of the serialized value + // bytes, so the combined-hash check below is identical. if !verify_references { continue; } diff --git a/grovedb/src/operations/get/mod.rs b/grovedb/src/operations/get/mod.rs index 66ab6b04b..764050d1e 100644 --- a/grovedb/src/operations/get/mod.rs +++ b/grovedb/src/operations/get/mod.rs @@ -72,6 +72,8 @@ impl GroveDb { // Look through `NonCounted` so a wrapped reference still resolves. // The wrapper is transparent at the get/query layer. + // `ReferenceWithSumItem` follows the same chain as `Reference` — the + // carried sum is a parent-aggregation property, not a per-hop value. match cost_return_on_error!( &mut cost, self.get_raw_caching_optional( @@ -84,7 +86,8 @@ impl GroveDb { ) .into_underlying() { - Element::Reference(reference_path, ..) => { + Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..) => { let path_owned = cost_return_on_error_into!( &mut cost, path_from_reference_path_type(reference_path, &path.to_vec(), Some(key)) @@ -162,8 +165,11 @@ impl GroveDb { visited.insert(current_path.clone()); // Look through `NonCounted` so a chain that hops via a wrapped // reference is followed instead of being returned as a value. + // `ReferenceWithSumItem` is also followed — the carried sum is + // irrelevant to chain destination. match current_element.into_underlying() { - Element::Reference(reference_path, ..) => { + Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..) => { current_path = cost_return_on_error_into!( &mut cost, path_from_reference_qualified_path_type(reference_path, ¤t_path) diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index 363379375..dd496e9b4 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -95,7 +95,9 @@ impl GroveDb { // resolves; the wrapper is transparent at the query // layer. match element.into_underlying() { - Element::Reference(reference_path, ..) => match reference_path { + Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..) => match reference_path + { ReferencePathType::AbsolutePathReference(absolute_path) => { // While `map` on iterator is lazy, we should accumulate costs // even if `collect` will end in `Err`, so we'll use @@ -226,7 +228,8 @@ where { // sole effect is on parent count aggregation. let element = element.into_underlying(); match element { - Element::Reference(reference_path, ..) => { + Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..) => { match reference_path { ReferencePathType::AbsolutePathReference(absolute_path) => { // While `map` on iterator is lazy, we should accumulate costs @@ -237,6 +240,9 @@ where { // Normalize the resolved value too, so a Reference // pointing at NonCounted(Item) returns the same shape // as a directly-queried NonCounted(Item). + // `ReferenceWithSumItem` follows the same resolution + // path; the sum carried on the source element does + // not affect what `follow_reference` returns. let maybe_item = self .follow_reference( absolute_path.as_slice().into(), @@ -364,7 +370,12 @@ where { // NonCounted is transparent at this layer. let element = element.into_underlying(); match element { - Element::Reference(reference_path, ..) => { + // `ReferenceWithSumItem` resolves to the target item + // the same way `Reference` does; the carried sum is + // ignored at this layer (use `query_item_value_or_sum` + // to see it). + Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..) => { match reference_path { ReferencePathType::AbsolutePathReference(absolute_path) => { // While `map` on iterator is lazy, we should accumulate costs @@ -468,7 +479,12 @@ where { // NonCounted is transparent at this layer. let element = element.into_underlying(); match element { - Element::Reference(reference_path, ..) => { + // `ReferenceWithSumItem` resolves to the target item + // exactly like `Reference`; the carried sum value + // does not show up here (it's an aggregate-only + // property that propagates to the parent tree). + Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..) => { match reference_path { ReferencePathType::AbsolutePathReference(absolute_path) => { // While `map` on iterator is lazy, we should accumulate costs @@ -961,7 +977,13 @@ where { // NonCounted is transparent at this layer. let element = element.into_underlying(); match element { - Element::Reference(reference_path, ..) => { + // For `ReferenceWithSumItem` we follow the reference + // just like `Reference` — the carried sum is a + // parent-aggregation property, not the queryable + // leaf value. Target must still be a SumItem for + // `query_sums` to succeed. + Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..) => { match reference_path { ReferencePathType::AbsolutePathReference(absolute_path) => { // While `map` on iterator is lazy, we should accumulate costs diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index b63a0fe74..5818ab040 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -246,7 +246,12 @@ impl GroveDb { // calls operate on the outer wrapper, which is what we want — the // serialized wrapper bytes go to storage. match element.underlying() { - Element::Reference(reference_path, ..) => { + // `ReferenceWithSumItem` shares the reference resolution + proof + // shape with `Reference`. The merk feature_type derived from the + // element's `sum_value_or_default()` already routes the sum into + // any sum-bearing parent; the call site is otherwise identical. + Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..) => { let path = path.to_vec(); // TODO: need for support for references in path library let reference_path = cost_return_on_error_into!( &mut cost, diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 386143093..b87a9d004 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -400,7 +400,16 @@ impl GroveDb { let elem = Element::deserialize(value, grove_version).map(|e| e.into_underlying()); match elem { - Ok(Element::Reference(reference_path, ..)) => { + // `ReferenceWithSumItem` shares the proof shape + // with `Reference`: combined value hash, GroveDB + // post-processes to KVRefValueHash{,Count} with + // the dereferenced value. The carried sum is + // hashed inside the (unchanged) serialized `value` + // bytes; the proof verifier sees it as part of + // the reference's KV-value-hash and the + // parent's feature_type (via merk's normal flow). + Ok(Element::Reference(reference_path, ..)) + | Ok(Element::ReferenceWithSumItem(reference_path, ..)) => { let absolute_path = cost_return_on_error_into!( &mut cost, path_from_reference_path_type( @@ -1165,7 +1174,12 @@ impl GroveDb { let elem = Element::deserialize(value, grove_version).map(|e| e.into_underlying()); match elem { - Ok(Element::Reference(reference_path, ..)) => { + // `ReferenceWithSumItem` shares this proof path + // with `Reference` — both produce a + // KVRefValueHash{,Count} node with the + // dereferenced target's serialized bytes. + Ok(Element::Reference(reference_path, ..)) + | Ok(Element::ReferenceWithSumItem(reference_path, ..)) => { let absolute_path = cost_return_on_error_into!( &mut cost, path_from_reference_path_type( diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index fb90757b1..0431e92c2 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -637,7 +637,8 @@ impl GroveDb { | Element::SumItem(..) | Element::Item(..) | Element::ItemWithSumItem(..) - | Element::Reference(..) => { + | Element::Reference(..) + | Element::ReferenceWithSumItem(..) => { return Err(Error::InvalidProof( query.clone(), "V1 proof has lower layer for a non-tree element.".to_string(), @@ -1616,7 +1617,8 @@ impl GroveDb { | Element::SumItem(..) | Element::Item(..) | Element::ItemWithSumItem(..) - | Element::Reference(..) => { + | Element::Reference(..) + | Element::ReferenceWithSumItem(..) => { return Err(Error::InvalidProof( query.clone(), "Proof has lower layer for a non Tree.".to_string(), diff --git a/grovedb/src/reference_path.rs b/grovedb/src/reference_path.rs index 1239306f2..0804e76e7 100644 --- a/grovedb/src/reference_path.rs +++ b/grovedb/src/reference_path.rs @@ -92,8 +92,12 @@ pub(crate) fn follow_reference<'db, 'b, 'c, B: AsRef<[u8]>>( // affect downstream cryptographic verification. `NotSummed` cannot // wrap a reference by construction (whitelist), but the unwrap is // forward-safe and symmetric to `NonCounted`. + // Both `Reference` and `ReferenceWithSumItem` are references — they + // share the resolution path. The carried sum on + // `ReferenceWithSumItem` is irrelevant to chain following: it's a + // parent-aggregation property, not a per-hop value. match element.into_underlying() { - Element::Reference(ref_path, ..) => { + Element::Reference(ref_path, ..) | Element::ReferenceWithSumItem(ref_path, ..) => { current_path = referred_path; current_key = referred_key; current_ref = ref_path; diff --git a/grovedb/src/tests/batch_coverage_tests.rs b/grovedb/src/tests/batch_coverage_tests.rs index 7da11cc2d..12a129dd4 100644 --- a/grovedb/src/tests/batch_coverage_tests.rs +++ b/grovedb/src/tests/batch_coverage_tests.rs @@ -109,6 +109,7 @@ mod tests { ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]), Some(5), None, + /* non_counted = */ false, true, )]; @@ -314,6 +315,7 @@ mod tests { ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]), Some(5), None, + /* non_counted = */ false, true, )]; @@ -1796,6 +1798,7 @@ mod tests { ref_path, Some(5), None, + /* non_counted = */ false, false, // untrusted: read from disk )]; @@ -2614,6 +2617,7 @@ mod tests { ]), Some(5), None, + /* non_counted = */ false, true, ), ]; diff --git a/grovedb/src/tests/batch_unit_tests.rs b/grovedb/src/tests/batch_unit_tests.rs index 8b2a94ce3..f7b0b0e09 100644 --- a/grovedb/src/tests/batch_unit_tests.rs +++ b/grovedb/src/tests/batch_unit_tests.rs @@ -108,8 +108,9 @@ mod tests { // 5 reference_path_type: ReferencePathType::AbsolutePathReference(vec![]), max_reference_hop: None, + mode: crate::batch::RefreshReferenceMode::PlainReferenceUntrusted, flags: None, - trust_refresh_reference: false, + non_counted: false, }, GroveOp::Replace { // 6 @@ -395,6 +396,7 @@ mod tests { ReferencePathType::AbsolutePathReference(vec![]), None, None, + /* non_counted = */ false, false, ), "Refresh Reference", @@ -1175,6 +1177,7 @@ mod tests { ]), None, None, + /* non_counted = */ false, false, )]; @@ -1394,6 +1397,7 @@ mod tests { ]), Some(2), None, + /* non_counted = */ false, true, // trust_refresh_reference ), QualifiedGroveDbOp::insert_or_replace_op( @@ -1460,6 +1464,7 @@ mod tests { ]), Some(2), None, + /* non_counted = */ false, false, // trust_refresh_reference = false ), QualifiedGroveDbOp::insert_or_replace_op( @@ -1585,6 +1590,7 @@ mod tests { ]), Some(2), None, + /* non_counted = */ false, true, ), QualifiedGroveDbOp::insert_or_replace_op( diff --git a/grovedb/src/tests/estimated_costs_average_case_tests.rs b/grovedb/src/tests/estimated_costs_average_case_tests.rs index 7898f3c02..0f74f47a8 100644 --- a/grovedb/src/tests/estimated_costs_average_case_tests.rs +++ b/grovedb/src/tests/estimated_costs_average_case_tests.rs @@ -127,6 +127,7 @@ fn test_average_case_merk_insert_tree_no_flags_no_propagate() { &flags, TreeType::NormalTree, TreeType::NormalTree, + 0, None, grove_version, ); @@ -150,6 +151,7 @@ fn test_average_case_merk_insert_tree_with_flags_and_propagate() { &flags, TreeType::NormalTree, TreeType::NormalTree, + 0, Some(&layer_info), grove_version, ); @@ -175,6 +177,7 @@ fn test_average_case_merk_insert_sum_tree() { &flags, TreeType::SumTree, TreeType::NormalTree, + 0, None, grove_version, ); diff --git a/grovedb/src/tests/estimated_costs_worst_case_tests.rs b/grovedb/src/tests/estimated_costs_worst_case_tests.rs index 94d0a3bb4..6f3b28cb1 100644 --- a/grovedb/src/tests/estimated_costs_worst_case_tests.rs +++ b/grovedb/src/tests/estimated_costs_worst_case_tests.rs @@ -107,6 +107,7 @@ fn test_worst_case_merk_insert_tree_no_flags_no_propagate() { &flags, TreeType::NormalTree, TreeType::NormalTree, + 0, None, grove_version, ); @@ -130,6 +131,7 @@ fn test_worst_case_merk_insert_tree_with_flags_and_propagate() { &flags, TreeType::NormalTree, TreeType::NormalTree, + 0, Some(&layer_info), grove_version, ); @@ -155,6 +157,7 @@ fn test_worst_case_merk_insert_sum_tree() { &flags, TreeType::SumTree, TreeType::NormalTree, + 0, None, grove_version, ); diff --git a/grovedb/src/tests/misc_coverage_tests.rs b/grovedb/src/tests/misc_coverage_tests.rs index 49659fa07..52b401be5 100644 --- a/grovedb/src/tests/misc_coverage_tests.rs +++ b/grovedb/src/tests/misc_coverage_tests.rs @@ -1944,6 +1944,7 @@ fn batch_average_case_refresh_reference_cost() { ]), Some(10), Some(b"flags".to_vec()), + /* non_counted = */ false, true, )]; @@ -2247,6 +2248,7 @@ fn batch_worst_case_refresh_reference_cost() { ]), Some(10), Some(b"flags".to_vec()), + /* non_counted = */ false, true, )]; diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index 1e59c7e2e..58ec31d90 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -42,6 +42,7 @@ mod provable_count_tree_structure_test; mod provable_count_tree_test; mod query_result_type_tests; mod reference_path_tests; +mod reference_with_sum_item_tests; mod replication_session_tests; mod replication_utils_tests; mod succinctness_gap_test; diff --git a/grovedb/src/tests/reference_with_sum_item_tests.rs b/grovedb/src/tests/reference_with_sum_item_tests.rs new file mode 100644 index 000000000..0f3facc93 --- /dev/null +++ b/grovedb/src/tests/reference_with_sum_item_tests.rs @@ -0,0 +1,2125 @@ +//! End-to-end tests for `Element::ReferenceWithSumItem` and the +//! sum-item variant of the unified `GroveOp::RefreshReference` batch +//! op (built via `QualifiedGroveDbOp::refresh_reference_with_sum_item_op`). +//! +//! The variant is a reference that ALSO carries an explicit `SumValue`. It +//! resolves like `Element::Reference` on `get()` (hop-limited, cycle-detected, +//! combined value hash) AND contributes its sum to a sum-bearing parent like +//! `SumItem` / `ItemWithSumItem`. The sum is independent of the resolved +//! target's value. +//! +//! Permitted in any parent tree type — in non-sum parents the carried sum is +//! silently dropped (same rule `ItemWithSumItem` follows). + +#[cfg(test)] +mod tests { + use grovedb_merk::{ + proofs::{query::QueryItem, Query}, + tree::AggregateData, + }; + use grovedb_version::version::GroveVersion; + + use crate::{ + batch::QualifiedGroveDbOp, + operations::get::QueryItemOrSumReturnType, + reference_path::ReferencePathType, + tests::{make_test_grovedb, TEST_LEAF}, + Element, GroveDb, PathQuery, + }; + + fn insert_target_item( + db: &crate::GroveDb, + parent_path: &[&[u8]], + key: &[u8], + bytes: &[u8], + grove_version: &GroveVersion, + ) { + db.insert( + parent_path, + key, + Element::new_item(bytes.to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert target item"); + } + + fn open_merk_aggregate( + db: &crate::GroveDb, + path: &[&[u8]], + grove_version: &GroveVersion, + ) -> AggregateData { + let transaction = db.start_transaction(); + let merk = db + .open_transactional_merk_at_path(path.into(), &transaction, None, grove_version) + .unwrap() + .expect("open merk"); + merk.aggregate_data().expect("aggregate data") + } + + /// Insert a `ReferenceWithSumItem` into a `SumTree` parent — its sum + /// propagates into the parent's running sum. + #[test] + fn insert_in_sum_tree_aggregates_sum() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + // Set up: SumTree under TEST_LEAF/st and a target Item elsewhere. + db.insert( + [TEST_LEAF].as_ref(), + b"st", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sum tree"); + + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"target", + b"target_payload", + grove_version, + ); + + // Reference-with-sum-item points to the target, carries sum 50. + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + let element = Element::new_reference_with_sum_item(ref_path, 50); + db.insert( + [TEST_LEAF, b"st"].as_ref(), + b"link", + element, + None, + None, + grove_version, + ) + .unwrap() + .expect("insert ref-with-sum-item"); + + // The parent SumTree should now total 50. + let agg = open_merk_aggregate(&db, &[TEST_LEAF, b"st"], grove_version); + assert_eq!(agg, AggregateData::Sum(50), "sum should propagate"); + } + + /// Two `ReferenceWithSumItem`s in the same SumTree both contribute their + /// (independent) sums. + #[test] + fn multiple_refs_with_sum_items_accumulate() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"st", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sum tree"); + + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target_a", b"a", grove_version); + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target_b", b"b", grove_version); + + for (key, target, sum) in [ + (b"link_a".as_ref(), b"target_a".as_ref(), 30i64), + (b"link_b".as_ref(), b"target_b".as_ref(), -8), + (b"link_c".as_ref(), b"target_a".as_ref(), 100), + ] { + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), target.to_vec()]); + db.insert( + [TEST_LEAF, b"st"].as_ref(), + key, + Element::new_reference_with_sum_item(ref_path, sum), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert link"); + } + + let agg = open_merk_aggregate(&db, &[TEST_LEAF, b"st"], grove_version); + assert_eq!(agg, AggregateData::Sum(30 + -8 + 100)); + } + + /// In a non-sum parent, the carried sum is silently dropped — same rule + /// `ItemWithSumItem` follows. + #[test] + fn insert_in_normal_tree_sum_silently_ignored() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + // TEST_LEAF is a normal tree — the carried sum has nowhere to go. + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"target", + b"target_payload", + grove_version, + ); + + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + db.insert( + [TEST_LEAF].as_ref(), + b"link", + Element::new_reference_with_sum_item(ref_path, 50), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert should succeed in NormalTree"); + + // No aggregate data for a normal tree. + let agg = open_merk_aggregate(&db, &[TEST_LEAF], grove_version); + assert_eq!(agg, AggregateData::NoAggregateData); + + // The reference is still resolvable to the target's bytes. + let resolved = db + .get([TEST_LEAF].as_ref(), b"link", None, grove_version) + .unwrap() + .expect("get link"); + assert_eq!( + resolved, + Element::new_item(b"target_payload".to_vec()), + "resolved value is the target item" + ); + } + + /// `get()` follows the reference to the target item. + #[test] + fn get_resolves_to_target_item_bytes() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"target", + b"payload", + grove_version, + ); + + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + db.insert( + [TEST_LEAF].as_ref(), + b"link", + Element::new_reference_with_sum_item(ref_path, 7), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert link"); + + let resolved = db + .get([TEST_LEAF].as_ref(), b"link", None, grove_version) + .unwrap() + .expect("get link"); + assert_eq!(resolved, Element::new_item(b"payload".to_vec())); + } + + /// `get_raw()` returns the new variant unfollowed, preserving the sum. + #[test] + fn get_raw_returns_reference_with_sum_item_unfollowed() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"target", + b"payload", + grove_version, + ); + + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + let element = Element::new_reference_with_sum_item(ref_path.clone(), 21); + db.insert( + [TEST_LEAF].as_ref(), + b"link", + element.clone(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert link"); + + let raw = db + .get_raw([TEST_LEAF].as_ref().into(), b"link", None, grove_version) + .unwrap() + .expect("get_raw link"); + assert_eq!(raw, element, "get_raw must return the variant verbatim"); + } + + /// Two-hop chain: `ReferenceWithSumItem` → `Reference` → `Item`. + /// `get()` resolves through both reference variants to the terminal + /// item, exercising the shared chain-follow match arm in + /// [`crate::operations::get::GroveDb::follow_reference`]. + #[test] + fn chain_through_reference_with_sum_item_resolves_to_terminal_item() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + // Terminal item. + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"terminal", + b"final_payload", + grove_version, + ); + + // Middle hop: plain Reference → terminal. + let to_terminal = ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"terminal".to_vec(), + ]); + db.insert( + [TEST_LEAF].as_ref(), + b"middle", + Element::new_reference(to_terminal), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert middle"); + + // First hop: ReferenceWithSumItem → middle. Both hops are + // exercised by `get()`. + let to_middle = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"middle".to_vec()]); + db.insert( + [TEST_LEAF].as_ref(), + b"head", + Element::new_reference_with_sum_item(to_middle, 99), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert head"); + + let resolved = db + .get([TEST_LEAF].as_ref(), b"head", None, grove_version) + .unwrap() + .expect("get head"); + assert_eq!(resolved, Element::new_item(b"final_payload".to_vec())); + } + + /// `NonCounted(ReferenceWithSumItem(_, _, sum, _))` in a `CountSumTree` + /// parent zeros the count contribution but still propagates the sum. + /// Mirrors `non_counted_tests.rs` expectations for other base variants. + #[test] + fn non_counted_reference_with_sum_item_in_count_sum_tree() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + // CountSumTree parent. + db.insert( + [TEST_LEAF].as_ref(), + b"cst", + Element::empty_count_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert count-sum tree"); + + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"target", + b"payload", + grove_version, + ); + + // First insert: bare ReferenceWithSumItem → contributes (1, sum). + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + db.insert( + [TEST_LEAF, b"cst"].as_ref(), + b"bare_link", + Element::new_reference_with_sum_item(ref_path.clone(), 25), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert bare link"); + + // Second insert: NonCounted wrapper → contributes (0, sum) so count + // stays at 1 (only `bare_link`) but sum totals 25 + 75 = 100. + let nc = Element::new_non_counted(Element::new_reference_with_sum_item(ref_path, 75)) + .expect("wrap ok"); + db.insert( + [TEST_LEAF, b"cst"].as_ref(), + b"nc_link", + nc, + None, + None, + grove_version, + ) + .unwrap() + .expect("insert nc link"); + + let agg = open_merk_aggregate(&db, &[TEST_LEAF, b"cst"], grove_version); + assert_eq!(agg, AggregateData::CountAndSum(1, 100)); + } + + /// Inserting a `ReferenceWithSumItem` via a batch (insert-or-replace op) + /// produces the same parent-sum aggregate as the direct insert path. + #[test] + fn batch_insert_reference_with_sum_item_propagates_sum() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"st", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sum tree"); + + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"target", + b"payload", + grove_version, + ); + + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + let op = QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"st".to_vec()], + b"link".to_vec(), + Element::new_reference_with_sum_item(ref_path, 50), + ); + db.apply_batch(vec![op], None, None, grove_version) + .unwrap() + .expect("batch apply"); + + let agg = open_merk_aggregate(&db, &[TEST_LEAF, b"st"], grove_version); + assert_eq!(agg, AggregateData::Sum(50)); + } + + /// Structural regression test for the unified + /// `GroveOp::RefreshReference` variant + the + /// [`RefreshReferenceMode`] enum that encodes both on-disk shape + /// and trust mode in 5 variants. The (NoValueUpdate, trusted) + /// combination doesn't exist by construction. + /// + /// All three public constructors (`refresh_reference_op`, + /// `refresh_reference_with_sum_item_op`, + /// `refresh_reference_with_sum_item_keep_sum_op`) must build the + /// same `GroveOp::RefreshReference`, distinguished only by the + /// `mode` variant. + #[test] + fn refresh_reference_constructors_share_unified_variant() { + use crate::batch::{GroveOp, RefreshReferenceMode}; + + let ref_path = ReferencePathType::AbsolutePathReference(vec![b"target".to_vec()]); + + let plain_trusted = QualifiedGroveDbOp::refresh_reference_op( + vec![TEST_LEAF.to_vec()], + b"link".to_vec(), + ref_path.clone(), + Some(2), + None, + /* non_counted = */ true, // <- exercise the new parameter + /* trust_refresh_reference = */ true, + ) + .op; + + let plain_untrusted = QualifiedGroveDbOp::refresh_reference_op( + vec![TEST_LEAF.to_vec()], + b"link".to_vec(), + ref_path.clone(), + Some(2), + None, + /* non_counted = */ false, + /* trust_refresh_reference = */ false, + ) + .op; + + let with_sum_trusted = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec()], + b"link".to_vec(), + ref_path.clone(), + Some(2), + 42, + None, + /* non_counted = */ true, + /* trust_refresh_reference = */ true, + ) + .op; + + let with_sum_untrusted = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec()], + b"link".to_vec(), + ref_path.clone(), + Some(2), + 42, + None, + /* non_counted = */ false, + /* trust_refresh_reference = */ false, + ) + .op; + + let keep_sum = QualifiedGroveDbOp::refresh_reference_with_sum_item_keep_sum_op( + vec![TEST_LEAF.to_vec()], + b"link".to_vec(), + ref_path, + Some(2), + None, + /* non_counted = */ false, + ) + .op; + + // All five are `GroveOp::RefreshReference`. + for op in [ + &plain_trusted, + &plain_untrusted, + &with_sum_trusted, + &with_sum_untrusted, + &keep_sum, + ] { + assert!( + matches!(op, GroveOp::RefreshReference { .. }), + "all constructors must build unified RefreshReference; got {op:?}" + ); + } + + // `mode` discriminates the five shapes. + let mode_of = |op: &GroveOp| -> RefreshReferenceMode { + let GroveOp::RefreshReference { mode, .. } = op else { + unreachable!() + }; + mode.clone() + }; + assert_eq!( + mode_of(&plain_trusted), + RefreshReferenceMode::PlainReferenceTrusted, + ); + assert_eq!( + mode_of(&plain_untrusted), + RefreshReferenceMode::PlainReferenceUntrusted, + ); + assert_eq!( + mode_of(&with_sum_trusted), + RefreshReferenceMode::SumItemReferenceTrusted(42), + ); + assert_eq!( + mode_of(&with_sum_untrusted), + RefreshReferenceMode::SumItemReferenceUntrustedValueUpdate(42), + ); + assert_eq!( + mode_of(&keep_sum), + RefreshReferenceMode::SumItemReferenceUntrustedNoValueUpdate, + ); + + // `is_trusted` helper agrees. + assert!(mode_of(&plain_trusted).is_trusted()); + assert!(!mode_of(&plain_untrusted).is_trusted()); + assert!(mode_of(&with_sum_trusted).is_trusted()); + assert!(!mode_of(&with_sum_untrusted).is_trusted()); + assert!(!mode_of(&keep_sum).is_trusted()); + + // `non_counted` is threaded through by all three constructors. + let non_counted_of = |op: &GroveOp| -> bool { + let GroveOp::RefreshReference { non_counted, .. } = op else { + unreachable!() + }; + *non_counted + }; + assert!( + non_counted_of(&plain_trusted), + "refresh_reference_op must thread non_counted=true through" + ); + assert!( + !non_counted_of(&plain_untrusted), + "refresh_reference_op with non_counted=false must NOT set the wrapper bit" + ); + assert!(non_counted_of(&with_sum_trusted)); + assert!(!non_counted_of(&with_sum_untrusted)); + assert!(!non_counted_of(&keep_sum)); + } + + /// The sum-item variant of `RefreshReference` updates the link AND the + /// sum atomically. + /// The parent SumTree must reflect the delta (new_sum - old_sum). + #[test] + fn batch_refresh_reference_with_sum_item_updates_sum_and_path() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"st", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sum tree"); + + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target_a", b"a", grove_version); + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target_b", b"b", grove_version); + + // Initial insert: link → target_a, sum 10. + let ref_a = ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"target_a".to_vec(), + ]); + db.insert( + [TEST_LEAF, b"st"].as_ref(), + b"link", + Element::new_reference_with_sum_item(ref_a, 10), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert"); + assert_eq!( + open_merk_aggregate(&db, &[TEST_LEAF, b"st"], grove_version), + AggregateData::Sum(10) + ); + + // Refresh: link → target_b, sum 25. Use trust_refresh_reference = true + // so we don't need the element on disk to be a `Reference` already. + let ref_b = ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"target_b".to_vec(), + ]); + let refresh = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec(), b"st".to_vec()], + b"link".to_vec(), + ref_b.clone(), + None, + 25, + None, + /* non_counted = */ false, + /* trust_refresh_reference = */ true, + ); + db.apply_batch(vec![refresh], None, None, grove_version) + .unwrap() + .expect("apply refresh"); + + // Sum moved by +15. + assert_eq!( + open_merk_aggregate(&db, &[TEST_LEAF, b"st"], grove_version), + AggregateData::Sum(25) + ); + + // The stored variant still resolves to the new target. + let resolved = db + .get([TEST_LEAF, b"st"].as_ref(), b"link", None, grove_version) + .unwrap() + .expect("get refreshed link"); + assert_eq!(resolved, Element::new_item(b"b".to_vec())); + + // get_raw confirms the new sum is on disk. + let raw = db + .get_raw( + [TEST_LEAF, b"st"].as_ref().into(), + b"link", + None, + grove_version, + ) + .unwrap() + .expect("get_raw refreshed link"); + assert_eq!(raw, Element::new_reference_with_sum_item(ref_b, 25)); + } + + /// Applying `RefreshReferenceWithSumItem` against a plain `Reference` on + /// disk (with `trust_refresh_reference=false`) must be rejected — silent + /// coercion would corrupt the parent's aggregate. + #[test] + fn batch_refresh_cross_type_rejected() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"st", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sum tree"); + + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target", b"x", grove_version); + + // Insert a *plain* Reference (no sum) on disk. + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + db.insert( + [TEST_LEAF, b"st"].as_ref(), + b"link", + Element::new_reference(ref_path.clone()), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert"); + + // Refresh as RefreshReferenceWithSumItem (no trust). The apply path + // must reject because the on-disk variant is `Reference`, not + // `ReferenceWithSumItem`. + let refresh = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec(), b"st".to_vec()], + b"link".to_vec(), + ref_path, + None, + 42, + None, + /* non_counted = */ false, + /* trust_refresh_reference = */ false, + ); + let err = db + .apply_batch(vec![refresh], None, None, grove_version) + .unwrap() + .expect_err("cross-type refresh must fail"); + let msg = format!("{err:?}"); + assert!( + msg.contains("RefWithSumItem") + || msg.contains("ReferenceWithSumItem") + || msg.contains("non-RefWithSumItem"), + "expected cross-type rejection error, got: {msg}" + ); + } + + /// Contract pin: `RefreshReference` with `trust=true` against a + /// `ReferenceWithSumItem` on disk **silently coerces** it to a + /// plain `Reference` — the carried sum is dropped and the parent + /// SumTree's aggregate becomes inconsistent. Documented behavior + /// of the trusted mode: the caller is asserting the on-disk + /// variant and accepts the consequences. + /// + /// This is NOT a bug. If a future contributor adds cross-type + /// validation to the trusted path, this test will fail and force + /// them to reconsider the contract. + #[test] + fn batch_refresh_reference_trusted_silently_coerces_ref_with_sum_item() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"st", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sum tree"); + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target", b"x", grove_version); + + // Seed a ReferenceWithSumItem with sum=10. Parent SumTree + // aggregate is +10. + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + db.insert( + [TEST_LEAF, b"st"].as_ref(), + b"link", + Element::new_reference_with_sum_item(ref_path.clone(), 10), + None, + None, + grove_version, + ) + .unwrap() + .expect("seed link"); + assert_eq!( + open_merk_aggregate(&db, &[TEST_LEAF, b"st"], grove_version), + AggregateData::Sum(10), + ); + + // RefreshReference with trust=true. The apply path writes the + // op's payload as a plain `Element::Reference(...)` without + // checking on-disk. The sum is silently dropped. + let refresh = QualifiedGroveDbOp::refresh_reference_op( + vec![TEST_LEAF.to_vec(), b"st".to_vec()], + b"link".to_vec(), + ref_path.clone(), + None, + None, + /* non_counted = */ false, + /* trust_refresh_reference = */ true, + ); + db.apply_batch(vec![refresh], None, None, grove_version) + .unwrap() + .expect("trusted refresh succeeds (silent coercion)"); + + // On-disk is now a plain Reference (sum dropped). + let raw = db + .get_raw( + [TEST_LEAF, b"st"].as_ref().into(), + b"link", + None, + grove_version, + ) + .unwrap() + .expect("get_raw"); + assert!( + matches!(raw, Element::Reference(..)), + "trust=true must overwrite with the op's declared variant; got {raw:?}", + ); + } + + /// Contract pin (mirror of the above): `RefreshReferenceWithSumItem` + /// with `trust=true` against a plain `Reference` on disk + /// **silently coerces** it to a `ReferenceWithSumItem` carrying the + /// op's `sum_value`. The parent SumTree's aggregate jumps by + /// `+sum_value`. Caller's responsibility. + #[test] + fn batch_refresh_reference_with_sum_item_trusted_silently_coerces_plain_reference() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"st", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sum tree"); + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target", b"x", grove_version); + + // Seed a plain Reference (no sum). Parent aggregate is 0. + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + db.insert( + [TEST_LEAF, b"st"].as_ref(), + b"link", + Element::new_reference(ref_path.clone()), + None, + None, + grove_version, + ) + .unwrap() + .expect("seed plain reference"); + assert_eq!( + open_merk_aggregate(&db, &[TEST_LEAF, b"st"], grove_version), + AggregateData::Sum(0), + ); + + // RefreshReferenceWithSumItem with trust=true. Apply writes + // the op's full payload without a disk read. + let refresh = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec(), b"st".to_vec()], + b"link".to_vec(), + ref_path, + None, + 77, + None, + /* non_counted = */ false, + /* trust_refresh_reference = */ true, + ); + db.apply_batch(vec![refresh], None, None, grove_version) + .unwrap() + .expect("trusted refresh succeeds (silent coercion)"); + + // On-disk is now a ReferenceWithSumItem(sum=77). + let raw = db + .get_raw( + [TEST_LEAF, b"st"].as_ref().into(), + b"link", + None, + grove_version, + ) + .unwrap() + .expect("get_raw"); + match raw { + Element::ReferenceWithSumItem(_, _, sum, _) => assert_eq!(sum, 77), + other => panic!("expected ReferenceWithSumItem after coercion, got {other:?}"), + } + } + + /// `is_reference` and `is_reference_with_sum_item` predicates work in + /// the end-to-end pipeline (post-deserialization). + #[test] + fn predicates_persist_through_round_trip() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"target", + b"payload", + grove_version, + ); + + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + let element = Element::new_reference_with_sum_item(ref_path, 11); + db.insert( + [TEST_LEAF].as_ref(), + b"link", + element, + None, + None, + grove_version, + ) + .unwrap() + .expect("insert"); + + let raw = db + .get_raw([TEST_LEAF].as_ref().into(), b"link", None, grove_version) + .unwrap() + .expect("get_raw"); + assert!(raw.is_reference()); + assert!(raw.is_reference_with_sum_item()); + assert!(!raw.is_any_item()); + assert_eq!(raw.sum_value_or_default(), 11); + } + + /// Inserting a `NotSummed(ReferenceWithSumItem(..))` is rejected at + /// construction — `NotSummed` only wraps sum-tree variants, not + /// reference-like leaves. + #[test] + fn new_not_summed_rejects_reference_with_sum_item() { + let ref_path = ReferencePathType::AbsolutePathReference(vec![b"a".to_vec()]); + let inner = Element::new_reference_with_sum_item(ref_path, 1); + assert!(Element::new_not_summed(inner).is_err()); + } + + /// Pins both refresh-reference constructor shapes to op-tag + /// `5` (the unified `GroveOp::RefreshReference` sort tag). + /// After `RefreshReferenceWithSumItem` was merged into + /// `RefreshReference` (with `sum_value: Option`), both + /// the plain and sum-item constructors must build the same op + /// variant and therefore share the same `to_u8`. The previous + /// dedicated tag `17` is now unused — leaving it as a "do not + /// reuse" hole avoids accidentally reassigning it to a new op. + #[test] + fn refresh_reference_op_tag_pin() { + use std::cmp::Ordering; + + let ref_path = ReferencePathType::AbsolutePathReference(vec![b"a".to_vec()]); + let refresh_sum = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![b"p".to_vec()], + b"k".to_vec(), + ref_path.clone(), + None, + 5, + None, + false, + true, + ) + .op; + let refresh_plain = QualifiedGroveDbOp::refresh_reference_op( + vec![b"p".to_vec()], + b"k".to_vec(), + ref_path, + None, + None, + /* non_counted = */ false, + true, + ) + .op; + + // Both constructors produce the unified GroveOp::RefreshReference + // with tag 5. + assert_eq!( + refresh_sum.to_u8(), + 5, + "refresh_reference_with_sum_item_op must build the unified RefreshReference (tag 5)", + ); + assert_eq!( + refresh_plain.to_u8(), + 5, + "refresh_reference_op must build the unified RefreshReference (tag 5)", + ); + + // Sanity: relative ordering against other ops continues to + // match the documented sort hierarchy. + let delete = QualifiedGroveDbOp::delete_op(vec![b"p".to_vec()], b"k".to_vec()).op; + let insert = QualifiedGroveDbOp::insert_or_replace_op( + vec![b"p".to_vec()], + b"k".to_vec(), + Element::new_item(b"x".to_vec()), + ) + .op; + assert_eq!(delete.cmp(&refresh_sum), Ordering::Less); + assert_eq!(insert.cmp(&refresh_sum), Ordering::Greater); + assert_eq!(refresh_sum.cmp(&refresh_sum.clone()), Ordering::Equal); + } + + /// Debug formatter for the unified `GroveOp::RefreshReference` + /// produces a string containing the path, max_hop, and mode — + /// exercises the `fmt::Debug` arm for the trusted sum-item shape. + /// The op-name label switches between "Refresh Reference" and + /// "Refresh Reference With Sum Item" depending on the + /// [`RefreshReferenceMode`]. Trust mode is encoded in the mode + /// variant name (no separate `trust_reference` field). + #[test] + fn refresh_reference_with_sum_item_debug_format() { + let op = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![b"parent".to_vec()], + b"child".to_vec(), + ReferencePathType::AbsolutePathReference(vec![b"target".to_vec()]), + Some(3), + 42, + None, + false, + true, + ); + let s = format!("{op:?}"); + assert!( + s.contains("Refresh Reference With Sum Item"), + "Debug should include op name: {s}" + ); + assert!(s.contains("max_hop"), "Debug should mention max_hop: {s}"); + assert!( + s.contains("mode SumItemReferenceTrusted(42)"), + "Debug should include the mode + sum: {s}", + ); + } + + /// Trusted refresh with `non_counted = true` against a CountSumTree + /// parent goes through `Element::new_non_counted` on the rebuilt + /// inner. Reaffirms the wrap-on-write block in the trust=true path. + /// CountSumTree is both count- and sum-bearing, which is the only + /// parent that accepts NonCounted-wrapped reference variants. + #[test] + fn batch_refresh_reference_with_sum_item_trusted_with_nc_wraps() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"cst", + Element::empty_count_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert count-sum tree"); + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target", b"x", grove_version); + + // Seed link with NonCounted(RefWithSum(_, _, 8, _)). + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + let nc = + Element::new_non_counted(Element::new_reference_with_sum_item(ref_path.clone(), 8)) + .expect("wrap ok"); + db.insert( + [TEST_LEAF, b"cst"].as_ref(), + b"link", + nc, + None, + None, + grove_version, + ) + .unwrap() + .expect("seed nc link"); + assert_eq!( + open_merk_aggregate(&db, &[TEST_LEAF, b"cst"], grove_version), + AggregateData::CountAndSum(0, 8), + ); + + // Trusted refresh with non_counted=true rewraps in NonCounted. + let refresh = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec(), b"cst".to_vec()], + b"link".to_vec(), + ref_path, + None, + 33, + None, + /* non_counted = */ true, + /* trust_refresh_reference = */ true, + ); + db.apply_batch(vec![refresh], None, None, grove_version) + .unwrap() + .expect("apply trusted refresh"); + + assert_eq!( + open_merk_aggregate(&db, &[TEST_LEAF, b"cst"], grove_version), + AggregateData::CountAndSum(0, 33), + ); + // Wrapper preserved on disk. + let raw = db + .get_raw( + [TEST_LEAF, b"cst"].as_ref().into(), + b"link", + None, + grove_version, + ) + .unwrap() + .expect("get_raw"); + assert!(raw.is_non_counted()); + } + + /// Untrusted refresh against a NonCounted(RefWithSum) with + /// `non_counted = true` succeeds — the disk shape matches the + /// declaration and the wrapper is preserved. + #[test] + fn batch_refresh_reference_with_sum_item_untrusted_matches_wrapper_succeeds() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"cst", + Element::empty_count_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert count-sum tree"); + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target", b"x", grove_version); + + // Seed wrapped variant. + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + let nc = + Element::new_non_counted(Element::new_reference_with_sum_item(ref_path.clone(), 1)) + .expect("wrap ok"); + db.insert( + [TEST_LEAF, b"cst"].as_ref(), + b"link", + nc, + None, + None, + grove_version, + ) + .unwrap() + .expect("seed"); + + // Untrusted refresh with matching non_counted=true. + let refresh = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec(), b"cst".to_vec()], + b"link".to_vec(), + ref_path, + None, + 7, + None, + /* non_counted = */ true, + /* trust_refresh_reference = */ false, + ); + db.apply_batch(vec![refresh], None, None, grove_version) + .unwrap() + .expect("apply untrusted refresh"); + + assert_eq!( + open_merk_aggregate(&db, &[TEST_LEAF, b"cst"], grove_version), + AggregateData::CountAndSum(0, 7), + ); + let raw = db + .get_raw( + [TEST_LEAF, b"cst"].as_ref().into(), + b"link", + None, + grove_version, + ) + .unwrap() + .expect("get_raw"); + assert!(raw.is_non_counted()); + } + + /// Regression test for the wrapper-invariant bypass: a trusted + /// `RefreshReferenceWithSumItem` with `non_counted = true` in a + /// non-count-bearing parent (here, a `NormalTree` under TEST_LEAF) + /// must be rejected. Without the apply-path guard, the trusted + /// branch would build `NonCounted(...)` and persist it into the + /// wrong tree type — violating the invariant that NonCounted-wrapped + /// elements only live in count-bearing trees. + #[test] + fn batch_refresh_reference_with_sum_item_trusted_with_nc_rejected_in_normal_tree() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target", b"x", grove_version); + + // Seed bare ReferenceWithSumItem in a NormalTree (TEST_LEAF). + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + db.insert( + [TEST_LEAF].as_ref(), + b"link", + Element::new_reference_with_sum_item(ref_path.clone(), 1), + None, + None, + grove_version, + ) + .unwrap() + .expect("seed bare ref-with-sum-item"); + + // Trusted refresh with non_counted=true. Without the guard the + // apply path would build NonCounted(...) and write it to disk + // under TEST_LEAF (NormalTree), silently violating the invariant. + let refresh = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec()], + b"link".to_vec(), + ref_path, + None, + 5, + None, + /* non_counted = */ true, + /* trust_refresh_reference = */ true, + ); + let err = db + .apply_batch(vec![refresh], None, None, grove_version) + .unwrap() + .expect_err("trusted refresh with non_counted=true in normal tree must fail"); + let msg = format!("{err:?}"); + assert!( + msg.contains("count-bearing") || msg.contains("non_counted"), + "expected wrapper-invariant rejection, got: {msg}" + ); + + // And the on-disk shape is unchanged — still bare, count is 1. + let raw = db + .get_raw([TEST_LEAF].as_ref().into(), b"link", None, grove_version) + .unwrap() + .expect("get_raw"); + assert!(!raw.is_non_counted(), "wrapper must not have been written"); + } + + /// Regression test: a dependent reference re-inserted in the same + /// batch as a trusted `RefreshReferenceWithSumItem` of its target + /// must commit its value_hash against the **refreshed** target, + /// not the stale on-disk one. Uses `trust=true` because that's the + /// mode where apply rewrites the path — under `trust=false` the + /// apply path keeps the on-disk path (see + /// `batch_untrusted_refresh_keeps_on_disk_path_only_sum_updates`). + /// + /// The `RefreshReference[WithSumItem]` arm in + /// `follow_reference_get_value_hash` gates the path threaded into + /// `process_reference` on `trust_refresh_reference`: `Some(op_path)` + /// when trusted, `None` when not. This keeps the dependent-ref + /// resolution consistent with whichever path the apply step will + /// actually write. + #[test] + fn batch_dependent_ref_resolves_through_refreshed_path_via_chain() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"target_a", + b"AAAA", + grove_version, + ); + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"target_b", + b"BBBB", + grove_version, + ); + + // `link` is a ReferenceWithSumItem currently pointing at target_a. + let to_a = ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"target_a".to_vec(), + ]); + db.insert( + [TEST_LEAF].as_ref(), + b"link", + Element::new_reference_with_sum_item(to_a, 1), + None, + None, + grove_version, + ) + .unwrap() + .expect("seed link"); + + // `dep` is a plain Reference → link. No max_hop set (defaults + // to MAX_REFERENCE_HOPS) so the chain dep → link → target can + // resolve all the way to the terminal Item, which is the budget + // the direct insert path and `verify_grovedb` use. + let to_link = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"link".to_vec()]); + db.insert( + [TEST_LEAF].as_ref(), + b"dep", + Element::new_reference(to_link.clone()), + None, + None, + grove_version, + ) + .unwrap() + .expect("seed dep"); + + // Pre-batch: verify_grovedb should be clean. + let issues_before = db + .verify_grovedb(None, true, true, grove_version) + .expect("verify pre-batch"); + assert!( + issues_before.is_empty(), + "pre-batch verify should be clean, got: {issues_before:?}" + ); + + // Batch: refresh `link` to point at target_b AND re-insert `dep` + // so its merk-stored value_hash gets recomputed. After the fix, + // dep's stored hash must combine with target_b's simple hash + // (the chain's terminal), not link's old merk-combined hash. + let to_b = ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"target_b".to_vec(), + ]); + let refresh = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec()], + b"link".to_vec(), + to_b, + None, + 2, + None, + /* non_counted = */ false, + /* trust_refresh_reference = */ true, + ); + let dep_replace = QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"dep".to_vec(), + Element::new_reference(to_link), + ); + db.apply_batch(vec![refresh, dep_replace], None, None, grove_version) + .unwrap() + .expect("apply batch"); + + // User-facing get follows the chain at read time. + let resolved = db + .get([TEST_LEAF].as_ref(), b"dep", None, grove_version) + .unwrap() + .expect("get dep"); + assert_eq!( + resolved, + Element::new_item(b"BBBB".to_vec()), + "dep should resolve to target_b after refresh" + ); + + // Load-bearing check: verify_grovedb must NOT report any hash + // mismatches. Pre-fix this failed with a mismatch on [test_leaf, + // dep] because dep was committed against the stale link hash. + let issues_after = db + .verify_grovedb(None, true, true, grove_version) + .expect("verify post-batch"); + assert!( + issues_after.is_empty(), + "post-batch verify must be clean after the P1 fix; got: {issues_after:?}" + ); + } + + /// `RefreshReferenceWithSumItem` against a non-existing key with + /// `trust=false` errors out — exercises the "trying to refresh a + /// non existing reference" branch in the apply path. + #[test] + fn batch_refresh_reference_with_sum_item_untrusted_missing_key_errors() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + let ref_path = ReferencePathType::AbsolutePathReference(vec![b"x".to_vec()]); + let refresh = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec()], + b"does_not_exist".to_vec(), + ref_path, + None, + 1, + None, + /* non_counted = */ false, + /* trust_refresh_reference = */ false, + ); + let err = db + .apply_batch(vec![refresh], None, None, grove_version) + .unwrap() + .expect_err("refresh of non-existing key must fail"); + let msg = format!("{err:?}"); + assert!( + msg.contains("non existing") || msg.contains("not") || msg.contains("MissingReference"), + "expected missing-key error, got: {msg}" + ); + } + + /// `query_item_value` follows a `ReferenceWithSumItem` to the target + /// item bytes — same as `Reference`. Exercises the new arm in + /// [`crate::operations::get::query::GroveDb::query_item_value`]. + #[test] + fn query_item_value_follows_reference_with_sum_item() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"target", + b"payload", + grove_version, + ); + + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + db.insert( + [TEST_LEAF].as_ref(), + b"link", + Element::new_reference_with_sum_item(ref_path, 99), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert link"); + + let mut query = Query::new(); + query.insert_key(b"link".to_vec()); + let path_query = PathQuery::new_unsized(vec![TEST_LEAF.to_vec()], query); + + let (items, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("query_item_value should succeed"); + assert_eq!(items, vec![b"payload".to_vec()]); + } + + /// `query_item_value_or_sum` follows a `ReferenceWithSumItem` and + /// returns the target item (same shape as following a `Reference`). + #[test] + fn query_item_value_or_sum_follows_reference_with_sum_item() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + // SumItem must live inside a sum-bearing tree. + db.insert( + [TEST_LEAF].as_ref(), + b"st", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sum tree"); + db.insert( + [TEST_LEAF, b"st"].as_ref(), + b"target", + Element::new_sum_item(50), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sum item target"); + + let ref_path = ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"st".to_vec(), + b"target".to_vec(), + ]); + db.insert( + [TEST_LEAF].as_ref(), + b"link", + // Carried sum (999) is independent of target's sum (50); the + // query returns the target's sum. + Element::new_reference_with_sum_item(ref_path, 999), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert link"); + + let mut query = Query::new(); + query.insert_key(b"link".to_vec()); + let path_query = PathQuery::new_unsized(vec![TEST_LEAF.to_vec()], query); + + let (items_or_sums, _) = db + .query_item_value_or_sum(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("query_item_value_or_sum should succeed"); + assert_eq!(items_or_sums, vec![QueryItemOrSumReturnType::SumValue(50)]); + } + + /// `query_sums` follows a `ReferenceWithSumItem` chain to a `SumItem` + /// target and returns the **target's** sum, not the carried sum. + #[test] + fn query_sums_follows_reference_with_sum_item_to_sum_item() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"st", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sum tree"); + db.insert( + [TEST_LEAF, b"st"].as_ref(), + b"target", + Element::new_sum_item(77), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sum item"); + + let ref_path = ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"st".to_vec(), + b"target".to_vec(), + ]); + db.insert( + [TEST_LEAF].as_ref(), + b"link", + Element::new_reference_with_sum_item(ref_path, 999), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert link"); + + let mut query = Query::new(); + query.insert_key(b"link".to_vec()); + let path_query = PathQuery::new_unsized(vec![TEST_LEAF.to_vec()], query); + + let (sums, _) = db + .query_sums(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("query_sums should succeed"); + assert_eq!(sums, vec![77]); + } + + /// `query_encoded_many` (multi-path) resolves a `ReferenceWithSumItem` + /// to its terminal item — covers the multi-path query arm in `query.rs` + /// near line 98. + #[test] + #[allow(deprecated)] + fn query_encoded_many_follows_reference_with_sum_item() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"target", + b"payload", + grove_version, + ); + + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + db.insert( + [TEST_LEAF].as_ref(), + b"link", + Element::new_reference_with_sum_item(ref_path, 7), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert link"); + + let mut query = Query::new(); + query.insert_key(b"link".to_vec()); + let path_query = PathQuery::new_unsized(vec![TEST_LEAF.to_vec()], query); + + let items = db + .query_encoded_many(&[&path_query], true, true, true, None, grove_version) + .unwrap() + .expect("query_encoded_many should succeed"); + assert_eq!(items, vec![b"payload".to_vec()]); + } + + /// `RefreshReferenceWithSumItem` with `trust_refresh_reference = false` + /// is the "refresh the carried weight only" mode. The apply path + /// reads the on-disk element, cross-checks variant + wrapper, and + /// writes back with the on-disk path / max-hop / flags — only + /// `sum_value` is taken from the op. The op's `reference_path_type` + /// is intentionally ignored. + #[test] + fn batch_refresh_reference_with_sum_item_untrusted() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"st", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sum tree"); + + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target_a", b"a", grove_version); + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target_b", b"b", grove_version); + + let ref_a = ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"target_a".to_vec(), + ]); + db.insert( + [TEST_LEAF, b"st"].as_ref(), + b"link", + Element::new_reference_with_sum_item(ref_a.clone(), 10), + None, + None, + grove_version, + ) + .unwrap() + .expect("seed link"); + + // Refresh with trust=false. Pass `ref_b` for the path: it must + // be IGNORED. Only `sum_value=42` is taken from the op. + let ref_b = ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"target_b".to_vec(), + ]); + let refresh = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec(), b"st".to_vec()], + b"link".to_vec(), + ref_b, + None, + 42, + None, + /* non_counted = */ false, + /* trust_refresh_reference = */ false, + ); + db.apply_batch(vec![refresh], None, None, grove_version) + .unwrap() + .expect("untrusted refresh ref-with-sum-item should succeed"); + + // Path stayed `ref_a` (on-disk); sum updated to 42 from the op. + let raw = db + .get_raw( + [TEST_LEAF, b"st"].as_ref().into(), + b"link", + None, + grove_version, + ) + .unwrap() + .expect("get_raw refreshed link"); + assert_eq!(raw, Element::new_reference_with_sum_item(ref_a, 42)); + } + + /// `RefreshReferenceWithSumItem` with `non_counted=true` and + /// `trust_refresh_reference=true` rebuilds the element wrapped in + /// `NonCounted`. This locks in the wrapper-preservation behavior the + /// `non_counted` field on the op exists to provide. + #[test] + fn batch_refresh_reference_with_sum_item_trusted_preserves_non_counted_wrapper() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + // CountSumTree parent so count + sum aggregates are observable. + db.insert( + [TEST_LEAF].as_ref(), + b"cst", + Element::empty_count_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert count-sum tree"); + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target", b"x", grove_version); + + // Seed a NonCounted(ReferenceWithSumItem) with sum 10. Count + // contribution is 0 (NonCounted), sum contribution is 10. + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + let nc_initial = + Element::new_non_counted(Element::new_reference_with_sum_item(ref_path.clone(), 10)) + .expect("wrap ok"); + db.insert( + [TEST_LEAF, b"cst"].as_ref(), + b"link", + nc_initial, + None, + None, + grove_version, + ) + .unwrap() + .expect("seed nc link"); + assert_eq!( + open_merk_aggregate(&db, &[TEST_LEAF, b"cst"], grove_version), + AggregateData::CountAndSum(0, 10), + ); + + // Trusted refresh with non_counted=true must preserve the wrapper. + let refresh = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec(), b"cst".to_vec()], + b"link".to_vec(), + ref_path.clone(), + None, + 25, + None, + /* non_counted = */ true, + /* trust_refresh_reference = */ true, + ); + db.apply_batch(vec![refresh], None, None, grove_version) + .unwrap() + .expect("apply trusted refresh"); + + // Count stays 0 (still NonCounted), sum becomes 25. + assert_eq!( + open_merk_aggregate(&db, &[TEST_LEAF, b"cst"], grove_version), + AggregateData::CountAndSum(0, 25), + ); + // On-disk shape is still NonCounted(ReferenceWithSumItem(_, _, 25, _)). + let raw = db + .get_raw( + [TEST_LEAF, b"cst"].as_ref().into(), + b"link", + None, + grove_version, + ) + .unwrap() + .expect("get_raw"); + assert!(raw.is_non_counted(), "wrapper preserved after refresh"); + assert_eq!(raw.sum_value_or_default(), 25); + } + + /// `RefreshReferenceWithSumItem` with `trust_refresh_reference=false` + /// and `non_counted` flag disagreeing with disk is rejected — silent + /// wrapper drop or injection would corrupt the parent's count + /// aggregate. + #[test] + fn batch_refresh_reference_with_sum_item_untrusted_rejects_wrapper_mismatch() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target", b"x", grove_version); + + // Seed a BARE ReferenceWithSumItem (not wrapped) under TEST_LEAF. + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + db.insert( + [TEST_LEAF].as_ref(), + b"link", + Element::new_reference_with_sum_item(ref_path.clone(), 1), + None, + None, + grove_version, + ) + .unwrap() + .expect("seed bare ref-with-sum-item"); + + // Untrusted refresh with non_counted=true must reject because the + // on-disk element is bare. + let bad = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec()], + b"link".to_vec(), + ref_path, + None, + 7, + None, + /* non_counted = */ true, + /* trust_refresh_reference = */ false, + ); + let err = db + .apply_batch(vec![bad], None, None, grove_version) + .unwrap() + .expect_err("wrapper-mismatch refresh must fail"); + let msg = format!("{err:?}"); + assert!( + msg.contains("non_counted") || msg.contains("wrapper") || msg.contains("disagrees"), + "expected wrapper-mismatch rejection, got: {msg}" + ); + } + + /// Regression test for the "stale dependent reference" issue: when a + /// batch contains both a trusted `RefreshReferenceWithSumItem` op + /// (which writes the op's new path) AND another reference that + /// points at the same key, the dependent reference's value hash + /// must be computed against the **refreshed** target, not the stale + /// on-disk one. + /// + /// Uses `trust=true` because that's the only mode where the apply + /// path writes the op's `reference_path_type`. With `trust=false` + /// the apply path keeps the on-disk path (see the sibling test + /// `batch_untrusted_refresh_keeps_on_disk_path_only_sum_updates`). + #[test] + fn batch_dependent_reference_resolves_through_refreshed_path() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + // Two distinct items so we can prove the dependent ref tracks the + // post-batch path. + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"item_old", + b"OLD", + grove_version, + ); + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"item_new", + b"NEW", + grove_version, + ); + + // Seed: `link` is a ReferenceWithSumItem → item_old (sum 1). + // `dep` is a plain Reference → link → item_old. + let to_old = ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"item_old".to_vec(), + ]); + let to_link = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"link".to_vec()]); + db.insert( + [TEST_LEAF].as_ref(), + b"link", + Element::new_reference_with_sum_item(to_old.clone(), 1), + None, + None, + grove_version, + ) + .unwrap() + .expect("seed link"); + db.insert( + [TEST_LEAF].as_ref(), + b"dep", + Element::new_reference(to_link.clone()), + None, + None, + grove_version, + ) + .unwrap() + .expect("seed dep"); + + // Batch: refresh link → item_new (trust=true, the only mode + // where apply rewrites the path), AND re-insert dep so its + // value hash gets re-derived in the same batch. dep's hash + // must derive from item_new (NEW), not item_old. + let to_new = ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"item_new".to_vec(), + ]); + let refresh = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec()], + b"link".to_vec(), + to_new.clone(), + None, + 99, + None, + /* non_counted = */ false, + /* trust_refresh_reference = */ true, + ); + let dep_replace = QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"dep".to_vec(), + Element::new_reference(to_link), + ); + db.apply_batch(vec![refresh, dep_replace], None, None, grove_version) + .unwrap() + .expect("apply refresh+dep batch"); + + // After the batch, getting dep must follow link (now pointing at + // item_new) and return "NEW", not "OLD". This is the user-visible + // proof that the batch's internal hash computation used the + // refreshed path. + let resolved = db + .get([TEST_LEAF].as_ref(), b"dep", None, grove_version) + .unwrap() + .expect("get dep"); + assert_eq!( + resolved, + Element::new_item(b"NEW".to_vec()), + "dependent ref should resolve through the refreshed path" + ); + + // verify_grovedb must be clean: dep's stored value_hash combines + // against item_new (NEW)'s simple hash, matching what a fresh + // chain walk recomputes. + let issues = db + .verify_grovedb(None, true, true, grove_version) + .expect("verify"); + assert!( + issues.is_empty(), + "verify_grovedb must be clean post-batch; got: {issues:?}" + ); + } + + /// `RefreshReferenceWithSumItem` with `trust=false` is the + /// "refresh-the-weight-only" mode: the apply path reads the on-disk + /// element, keeps its path / max_hop / flags / wrapper, and only + /// overwrites `sum_value` from the op. The op's + /// `reference_path_type` etc. are intentionally ignored in this + /// mode — callers who don't know (or don't want to assert) the path + /// can pass anything. + #[test] + fn batch_untrusted_refresh_keeps_on_disk_path_only_sum_updates() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"item_old", + b"OLD", + grove_version, + ); + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"item_new", + b"NEW", + grove_version, + ); + + // Insert link under a SumTree so we can observe the parent's + // aggregate before / after. + db.insert( + [TEST_LEAF].as_ref(), + b"sums", + Element::new_sum_tree(None), + None, + None, + grove_version, + ) + .unwrap() + .expect("seed sum tree"); + + let to_old = ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"item_old".to_vec(), + ]); + let to_new = ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"item_new".to_vec(), + ]); + db.insert( + [TEST_LEAF, b"sums"].as_ref(), + b"link", + Element::new_reference_with_sum_item(to_old.clone(), 1), + None, + None, + grove_version, + ) + .unwrap() + .expect("seed link"); + + // Untrusted refresh: pass `to_new` and `99` for the sum. Only + // the sum should land on disk; the path must stay `to_old`. + let refresh = QualifiedGroveDbOp::refresh_reference_with_sum_item_op( + vec![TEST_LEAF.to_vec(), b"sums".to_vec()], + b"link".to_vec(), + to_new, // intentionally bogus under trust=false + None, + 99, + None, + /* non_counted = */ false, + /* trust_refresh_reference = */ false, + ); + db.apply_batch(vec![refresh], None, None, grove_version) + .unwrap() + .expect("apply untrusted refresh"); + + // Path stayed on-disk: resolving link follows to_old → "OLD". + let resolved = db + .get([TEST_LEAF, b"sums"].as_ref(), b"link", None, grove_version) + .unwrap() + .expect("get link"); + assert_eq!( + resolved, + Element::new_item(b"OLD".to_vec()), + "untrusted refresh must NOT repoint; op's reference_path_type \ + is ignored when trust=false" + ); + + // Sum updated: the carried sum_value on disk is now 99 (was 1). + let raw = db + .get_raw( + [TEST_LEAF, b"sums"].as_ref().into(), + b"link", + None, + grove_version, + ) + .unwrap() + .expect("get_raw link"); + match raw { + Element::ReferenceWithSumItem(_, _, sum, _) => assert_eq!(sum, 99), + other => panic!("expected ReferenceWithSumItem, got {other:?}"), + } + + // verify_grovedb must be clean. + let issues = db + .verify_grovedb(None, true, true, grove_version) + .expect("verify"); + assert!( + issues.is_empty(), + "verify_grovedb must be clean post-batch; got: {issues:?}" + ); + } + + /// `refresh_reference_with_sum_item_keep_sum_op` (mode = + /// `SumItemReferenceNoValueUpdate`) refreshes the on-disk + /// `value_hash` of a `ReferenceWithSumItem` without changing the + /// carried sum. The parent's running sum stays the same. + #[test] + fn batch_refresh_keep_sum_preserves_on_disk_sum() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"st", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sum tree"); + insert_target_item(&db, [TEST_LEAF].as_ref(), b"target", b"x", grove_version); + + // Seed a RefWithSumItem with sum=17. Parent aggregate = 17. + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + db.insert( + [TEST_LEAF, b"st"].as_ref(), + b"link", + Element::new_reference_with_sum_item(ref_path.clone(), 17), + None, + None, + grove_version, + ) + .unwrap() + .expect("seed link"); + assert_eq!( + open_merk_aggregate(&db, &[TEST_LEAF, b"st"], grove_version), + AggregateData::Sum(17), + ); + + let refresh = QualifiedGroveDbOp::refresh_reference_with_sum_item_keep_sum_op( + vec![TEST_LEAF.to_vec(), b"st".to_vec()], + b"link".to_vec(), + ref_path.clone(), + None, + None, + /* non_counted = */ false, + ); + db.apply_batch(vec![refresh], None, None, grove_version) + .unwrap() + .expect("keep-sum refresh succeeds"); + + // On-disk sum unchanged. + let raw = db + .get_raw( + [TEST_LEAF, b"st"].as_ref().into(), + b"link", + None, + grove_version, + ) + .unwrap() + .expect("get_raw link"); + assert_eq!(raw, Element::new_reference_with_sum_item(ref_path, 17)); + assert_eq!( + open_merk_aggregate(&db, &[TEST_LEAF, b"st"], grove_version), + AggregateData::Sum(17), + "parent's sum aggregate must not move under keep-sum refresh", + ); + } + + /// `prove_query` + `verify_query_with_options` round-trip on a + /// `ReferenceWithSumItem` — exercises the V1 proof generation / + /// verification arms for the new variant. + #[test] + fn prove_and_verify_reference_with_sum_item() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + insert_target_item( + &db, + [TEST_LEAF].as_ref(), + b"target", + b"target_payload", + grove_version, + ); + + let ref_path = + ReferencePathType::AbsolutePathReference(vec![TEST_LEAF.to_vec(), b"target".to_vec()]); + db.insert( + [TEST_LEAF].as_ref(), + b"link", + Element::new_reference_with_sum_item(ref_path, 7), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert ref-with-sum-item"); + + let path_query = PathQuery::new_unsized( + vec![TEST_LEAF.to_vec()], + Query { + items: vec![QueryItem::Key(b"link".to_vec())], + default_subquery_branch: Default::default(), + left_to_right: true, + conditional_subquery_branches: None, + add_parent_tree_on_subquery: false, + }, + ); + + let proof = db + .prove_query(&path_query, None, grove_version) + .unwrap() + .expect("prove ref-with-sum-item"); + + let (root_hash, result_set) = GroveDb::verify_query_with_options( + &proof, + &path_query, + grovedb_merk::proofs::query::VerifyOptions { + absence_proofs_for_non_existing_searched_keys: false, + verify_proof_succinctness: false, + include_empty_trees_in_result: false, + }, + grove_version, + ) + .expect("verify ref-with-sum-item proof"); + + let expected_root = db.grove_db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root, "root hash should match"); + assert_eq!(result_set.len(), 1, "proof should return 1 result"); + // The resolved value is the target item's payload (reference was + // followed in the proof post-processing step). + let (_path, key, element) = &result_set[0]; + assert_eq!(key, b"link"); + let element = element.as_ref().expect("element should be Some"); + match element { + Element::Item(bytes, _) => assert_eq!(bytes, b"target_payload"), + other => panic!("expected resolved target Item, got {:?}", other), + } + } +} diff --git a/grovedbg-types/Cargo.toml b/grovedbg-types/Cargo.toml index 2f11034f6..255510226 100644 --- a/grovedbg-types/Cargo.toml +++ b/grovedbg-types/Cargo.toml @@ -10,3 +10,6 @@ repository = "https://github.com/dashpay/grovedb" [dependencies] serde = { workspace = true } serde_with = { version = "3.9.0", features = ["base64"] } + +[dev-dependencies] +serde_json = "1" diff --git a/grovedbg-types/src/lib.rs b/grovedbg-types/src/lib.rs index f9c1d1f89..cb9e1e693 100644 --- a/grovedbg-types/src/lib.rs +++ b/grovedbg-types/src/lib.rs @@ -181,6 +181,16 @@ pub enum Element { element_flags: Option>, }, Reference(Reference), + /// A reference that also carries an explicit `i64` sum-item value + /// contributed to a sum-bearing parent (independent of the + /// resolved target's value). The `reference` field encodes the + /// same path-discriminant shape and `element_flags` as a plain + /// [`Element::Reference`]; `sum_item_value` is the additional + /// carried weight. + ReferenceWithSumItem { + reference: Reference, + sum_item_value: i64, + }, } #[serde_as] @@ -316,3 +326,44 @@ pub enum TreeFeatureType { pub struct ProveOptions { pub decrease_limit_on_empty_sub_query_result: bool, } + +#[cfg(test)] +mod tests { + use super::*; + + /// JSON wire round-trip pin for `Element::ReferenceWithSumItem`. + /// The variant carries a nested `Reference` (with the full + /// path-discriminant + element_flags shape) and an independent + /// `sum_item_value`. A future renumbering or rename of either + /// piece trips this test. + #[test] + fn reference_with_sum_item_json_round_trip_absolute() { + let element = Element::ReferenceWithSumItem { + reference: Reference::AbsolutePathReference { + path: vec![b"leaf".to_vec(), b"target".to_vec()], + element_flags: Some(vec![1, 2, 3]), + }, + sum_item_value: 1_000_000_000, + }; + let json = serde_json::to_string(&element).expect("serialize"); + let back: Element = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(back, element); + } + + /// Negative-sum and `None`-flags round-trip cleanly too. + /// Exercises a non-absolute reference path discriminant to lock + /// in the wire shape across the full set. + #[test] + fn reference_with_sum_item_json_round_trip_sibling_no_flags() { + let element = Element::ReferenceWithSumItem { + reference: Reference::SiblingReference { + sibling_key: b"sib".to_vec(), + element_flags: None, + }, + sum_item_value: -42, + }; + let json = serde_json::to_string(&element).expect("serialize"); + let back: Element = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(back, element); + } +} diff --git a/merk/src/element/get.rs b/merk/src/element/get.rs index 7e480907f..00d8750da 100644 --- a/merk/src/element/get.rs +++ b/merk/src/element/get.rs @@ -420,7 +420,9 @@ impl ElementFetchFromStoragePrivateExtensions for Element { .unwrap_or(0); let element_for_cost = element.as_ref().map(|e| e.underlying()); match element_for_cost { - Some(Element::Item(..)) | Some(Element::Reference(..)) => { + Some(Element::Item(..)) + | Some(Element::Reference(..)) + | Some(Element::ReferenceWithSumItem(..)) => { // while the loaded item might be a sum item, it is given for free // as it would be very hard to know in advance cost.storage_loaded_bytes = KV::value_byte_cost_size_for_key_and_value_lengths( @@ -531,7 +533,7 @@ impl ElementFetchFromStoragePrivateExtensions for Element { let wrapper_overhead = if element.is_wrapped() { 1u32 } else { 0 }; let element_for_cost = element.underlying(); match element_for_cost { - Element::Item(..) | Element::Reference(..) => { + Element::Item(..) | Element::Reference(..) | Element::ReferenceWithSumItem(..) => { // while the loaded item might be a sum item, it is given for free // as it would be very hard to know in advance cost.storage_loaded_bytes = KV::value_byte_cost_size_for_key_and_value_lengths(