feat(grovedb,merk): provable offset on ProvableCountTree / ProvableCountSumTree single-range queries

grovedb-version/src/version/merk_versions.rs

@@ -4,6 +4,7 @@ use versioned_feature_core::FeatureVersion;
 pub struct MerkVersions {
     pub batch: MerkBatchVersions,
     pub average_case_costs: MerkAverageCaseCostsVersions,
+    pub proof: MerkProofVersions,
 }
 
 #[derive(Clone, Debug, Default)]
@@ -18,3 +19,18 @@ pub struct MerkAverageCaseCostsVersions {
     pub add_average_case_merk_propagate: FeatureVersion,
     pub sum_tree_estimated_size: FeatureVersion,
 }
+
+/// Merk-level proof method versions.
+#[derive(Clone, Debug, Default)]
+pub struct MerkProofVersions {
+    /// `Merk::prove_count_offset_on_range` — offset-paginated proof
+    /// for a single range on a `ProvableCountTree` /
+    /// `ProvableCountSumTree`. Version 0 is the initial implementation
+    /// shipped in grove v3 alongside the V1 proof envelope; v1/v2 do
+    /// not call this method (V0 proofs reject offsets unconditionally,
+    /// so the count-offset path never enters their dispatch).
+    ///
+    /// Bump this if the prover's emitted op stream changes shape in a
+    /// way that requires a coordinated verifier update.
+    pub prove_count_offset_on_range: FeatureVersion,
+}

grovedb-version/src/version/v1.rs

@@ -9,7 +9,9 @@ use crate::version::{
         GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBQueryLimits,
         GroveDBReplicationVersions, GroveDBVersions,
     },
-    merk_versions::{MerkAverageCaseCostsVersions, MerkBatchVersions, MerkVersions},
+    merk_versions::{
+        MerkAverageCaseCostsVersions, MerkBatchVersions, MerkProofVersions, MerkVersions,
+    },
     GroveVersion,
 };
 
@@ -213,5 +215,15 @@ pub const GROVE_V1: GroveVersion = GroveVersion {
             add_average_case_merk_propagate: 0,
             sum_tree_estimated_size: 0,
         },
+        // `prove_count_offset_on_range` is implementation-version 0
+        // here too — but in grove v1 the V0 proof envelope rejects
+        // offsets unconditionally at the grovedb layer, so this
+        // method is never actually called from v1's prove path.
+        // The field is kept consistent across grove versions so the
+        // method's `check_merk_v0_with_cost!` gate doesn't accidentally
+        // trip if someone calls it directly from a v1 context.
+        proof: MerkProofVersions {
+            prove_count_offset_on_range: 0,
+        },
     },
 };

grovedb-version/src/version/v2.rs

@@ -9,7 +9,9 @@ use crate::version::{
         GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBQueryLimits,
         GroveDBReplicationVersions, GroveDBVersions,
     },
-    merk_versions::{MerkAverageCaseCostsVersions, MerkBatchVersions, MerkVersions},
+    merk_versions::{
+        MerkAverageCaseCostsVersions, MerkBatchVersions, MerkProofVersions, MerkVersions,
+    },
     GroveVersion,
 };
 
@@ -213,5 +215,12 @@ pub const GROVE_V2: GroveVersion = GroveVersion {
             add_average_case_merk_propagate: 1, // changed
             sum_tree_estimated_size: 1,         // changed
         },
+        // See the comment in v1.rs — `prove_count_offset_on_range` is
+        // not reachable from v2's prove path (V0 envelope rejects
+        // offsets), but the version field is kept consistent so a
+        // direct caller doesn't trip the version gate.
+        proof: MerkProofVersions {
+            prove_count_offset_on_range: 0,
+        },
     },
 };

grovedb-version/src/version/v3.rs

@@ -9,7 +9,9 @@ use crate::version::{
         GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBQueryLimits,
         GroveDBReplicationVersions, GroveDBVersions,
     },
-    merk_versions::{MerkAverageCaseCostsVersions, MerkBatchVersions, MerkVersions},
+    merk_versions::{
+        MerkAverageCaseCostsVersions, MerkBatchVersions, MerkProofVersions, MerkVersions,
+    },
     GroveVersion,
 };
 
@@ -213,5 +215,10 @@ pub const GROVE_V3: GroveVersion = GroveVersion {
             add_average_case_merk_propagate: 1,
             sum_tree_estimated_size: 1,
         },
+        proof: MerkProofVersions {
+            // Initial implementation; introduced alongside the V1
+            // proof envelope.
+            prove_count_offset_on_range: 0,
+        },
     },
 };

grovedb/src/operations/proof/generate.rs

@@ -181,7 +181,97 @@ impl GroveDb {
         }
     }
 
+    /// Helper for the top-level count-offset gate in
+    /// `prove_query_non_serialized_v{0,1}`. Opens the merk at
+    /// `path_query.path` and confirms its `tree_type` is one of the
+    /// two count-bearing flavors. Run only when the caller has set a
+    /// non-zero offset *and* the syntactic gate
+    /// (`validate_count_offset_paginated`) already passed.
+    ///
+    /// Why this lives at the top entry rather than only at the
+    /// leaf-level short-circuit: for an empty NormalTree at the
+    /// target path, the descent inside `prove_subqueries_v{0,1}`
+    /// hits the empty-tree arm and *doesn't* recurse into the leaf
+    /// merk, so the leaf-level tree-type check never fires. Doing it
+    /// here gives callers a clear up-front error in that case.
+    ///
+    /// Error contract: any failure to resolve `path_query.path` to an
+    /// eligible merk surfaces as `Error::InvalidQuery`. We don't
+    /// forward the raw `open_transactional_merk_at_path` error because
+    /// it can leak storage-layer specifics (missing-path,
+    /// path-not-a-tree, corrupted-link, etc.) — from the caller's
+    /// point of view all of those have the same actionable meaning
+    /// here: "you can't run a count-offset query against this path",
+    /// and the single `InvalidQuery` covers all of them uniformly.
+    /// Storage-layer or hardware-IO errors still flow through but get
+    /// classified the same way; that's acceptable because the
+    /// alternative — surfacing them as `MerkError` / `CorruptedData`
+    /// from a purely syntactic gate — gives callers an unstable
+    /// error contract that depends on whether the merk happens to
+    /// exist.
+    fn check_count_offset_target_tree_type(
+        &self,
+        path_query: &PathQuery,
+        grove_version: &GroveVersion,
+    ) -> CostResult<(), Error> {
+        use grovedb_merk::TreeType as MerkTreeType;
+        let mut cost = OperationCost::default();
+        let tx = self.start_transaction();
+        let path_slices: Vec<&[u8]> = path_query.path.iter().map(|p| p.as_slice()).collect();
+        let open_result = self
+            .open_transactional_merk_at_path(
+                path_slices.as_slice().into(),
+                &tx,
+                None,
+                grove_version,
+            )
+            .unwrap_add_cost(&mut cost);
+        let target = match open_result {
+            Ok(t) => t,
+            Err(_e) => {
+                return Err(Error::InvalidQuery(
+                    "count-offset paginated queries are only valid against \
+                     ProvableCountTree / ProvableCountSumTree merks; the target path \
+                     could not be resolved to an eligible merk",
+                ))
+                .wrap_with_cost(cost);
+            }
+        };
+        if !matches!(
+            target.tree_type,
+            MerkTreeType::ProvableCountTree | MerkTreeType::ProvableCountSumTree
+        ) {
+            return Err(Error::InvalidQuery(
+                "count-offset paginated queries are only valid against \
+                 ProvableCountTree / ProvableCountSumTree merks",
+            ))
+            .wrap_with_cost(cost);
+        }
+        Ok(()).wrap_with_cost(cost)
+    }
+
     /// V0: Generates a Merk-only proof without serialization.
+    ///
+    /// ╔══════════════════════════════════════════════════════════════════╗
+    /// ║                  ⚠⚠⚠  DO NOT MODIFY V0 PROOFS  ⚠⚠⚠               ║
+    /// ╠══════════════════════════════════════════════════════════════════╣
+    /// ║ V0 is a **shipped wire format**. Live grove versions v1 and v2   ║
+    /// ║ produce and verify V0 proofs in production (see                  ║
+    /// ║ `grovedb-version` — `prove_query_non_serialized: 0` for both).   ║
+    /// ║ ANY change to the bytes V0 produces — adding new accepted        ║
+    /// ║ query shapes, accepting offsets that were previously rejected,   ║
+    /// ║ emitting new node variants, anything — silently changes what     ║
+    /// ║ deployed validators accept and is a consensus-breaking change.   ║
+    /// ║                                                                   ║
+    /// ║ New proof features go on V1 (`prove_query_non_serialized_v1`     ║
+    /// ║ in this file, `verify_layer_proof_v1` in verify.rs) and a fresh  ║
+    /// ║ `GroveVersion` that selects them. The V0 entry points must keep  ║
+    /// ║ behaving exactly as they did when v1/v2 shipped, including       ║
+    /// ║ rejecting every input v1/v2 rejected.                            ║
+    /// ║                                                                   ║
+    /// ║ If you find yourself wanting to "just adjust" something here:    ║
+    /// ║ STOP. Add the feature to V1 and bump the grove version instead.  ║
+    /// ╚══════════════════════════════════════════════════════════════════╝
     pub(crate) fn prove_query_non_serialized_v0(
         &self,
         path_query: &PathQuery,
@@ -273,7 +363,19 @@ impl GroveDb {
     }
 
     /// Perform a pre-order traversal of the tree based on the provided
-    /// subqueries
+    /// subqueries.
+    ///
+    /// ╔══════════════════════════════════════════════════════════════════╗
+    /// ║                  ⚠⚠⚠  DO NOT MODIFY V0 PROOFS  ⚠⚠⚠               ║
+    /// ╠══════════════════════════════════════════════════════════════════╣
+    /// ║ This function produces V0 proof bytes that are consumed by      ║
+    /// ║ grove versions v1 and v2 in production. Any change to the       ║
+    /// ║ accepted query shapes, the emitted op stream, or the wrapper    ║
+    /// ║ envelope is a consensus-breaking change. Add new features on    ║
+    /// ║ V1 (`prove_subqueries_v1`) behind a fresh grove version         ║
+    /// ║ instead. See `prove_query_non_serialized_v0` for the full       ║
+    /// ║ rationale.                                                       ║
+    /// ╚══════════════════════════════════════════════════════════════════╝
     pub(crate) fn prove_subqueries(
         &self,
         path: Vec<&[u8]>,
@@ -380,6 +482,16 @@ impl GroveDb {
             .wrap_with_cost(cost);
         }
 
+        // NOTE: count-offset paginated proofs are intentionally NOT
+        // supported on V0. The V0 envelope is a shipped wire format
+        // (grove versions v1 and v2 produce it in production); adding
+        // new accepted query shapes here would be a consensus-breaking
+        // change for already-deployed validators. The
+        // `prove_query_non_serialized_v0` entry-point rejects
+        // non-zero offsets unconditionally, so this short-circuit
+        // never needed to fire — leaving it out keeps the V0 proof
+        // surface identical to what shipped.
+
         let mut merk_proof = cost_return_on_error!(
             &mut cost,
             self.generate_merk_proof(
@@ -1094,10 +1206,32 @@ impl GroveDb {
         let prove_options = prove_options.unwrap_or_default();
 
         if path_query.query.offset.is_some() && path_query.query.offset != Some(0) {
-            return Err(Error::InvalidQuery(
-                "proved path queries can not have offsets",
-            ))
-            .wrap_with_cost(cost);
+            // A non-zero offset is honored *only* if the surrounding
+            // query is an offset-paginated range query against a
+            // ProvableCountTree / ProvableCountSumTree (see
+            // `SizedQuery::validate_count_offset_paginated`).
+            //
+            // We do two checks here at the top entry:
+            //   1. Syntactic gate via `validate_count_offset_paginated`
+            //      (single range item, no subqueries, offset > 0).
+            //   2. Open the target leaf merk and confirm its
+            //      `tree_type` is one of the two allowed flavors.
+            //
+            // Step 2 has to be done at the top because the leaf-level
+            // short-circuit in `prove_subqueries_v1` only fires after
+            // the descent reaches the leaf — and for an empty
+            // NormalTree at the target path the descent's empty-tree
+            // arm decrements the limit and returns instead of
+            // recursing, so the leaf check would silently accept.
+            // Doing the merk-open here gives a clear up-front error
+            // for that case.
+            if let Err(e) = path_query.validate_count_offset_paginated() {
+                return Err(e).wrap_with_cost(cost);
+            }
+            cost_return_on_error!(
+                &mut cost,
+                self.check_count_offset_target_tree_type(path_query, grove_version)
+            );
         }
         if path_query.query.limit == Some(0) {
             return Err(Error::InvalidQuery(
@@ -1226,6 +1360,74 @@ impl GroveDb {
             .wrap_with_cost(cost);
         }
 
+        // Count-offset paginated short-circuit (v1 path). Mirror of the
+        // aggregate-count/sum branches. Only fires at the leaf level
+        // (path is the full path_query.path) and only when the caller
+        // requested a non-zero offset on a syntactically-eligible query.
+        // The tree-type check happens here — the syntactic gate at the
+        // top entry already ran, so a mismatched tree type is a
+        // hard-error case (the caller asked for count-offset pagination
+        // against something that isn't a count tree).
+        if path.len() == path_query.path.len() && path_query.has_non_zero_offset() {
+            use grovedb_merk::TreeType as MerkTreeType;
+            let inner_range = cost_return_on_error_no_add!(
+                cost,
+                path_query.validate_count_offset_paginated().cloned()
+            );
+            if !matches!(
+                subtree.tree_type,
+                MerkTreeType::ProvableCountTree | MerkTreeType::ProvableCountSumTree
+            ) {
+                return Err(Error::InvalidQuery(
+                    "count-offset paginated queries are only valid against \
+                     ProvableCountTree / ProvableCountSumTree merks",
+                ))
+                .wrap_with_cost(cost);
+            }
+            let offset = path_query.query.offset.map(|o| o as u64).unwrap_or(0);
+            // Carry the SizedQuery::limit into the merk-level proof so
+            // the prover stops emitting value nodes once the requested
+            // page is full. After the merk prover returns, decrement
+            // the outer overall_limit accordingly so the upstream
+            // multi-layer accounting (if any) reflects the consumed
+            // slots.
+            let limit_u64 = path_query.query.limit.map(|l| l as u64);
+            let prove_result = cost_return_on_error!(
+                &mut cost,
+                subtree
+                    .prove_count_offset_on_range(
+                        &inner_range,
+                        offset,
+                        limit_u64,
+                        query.left_to_right,
+                        grove_version,
+                    )
+                    // Wrap with operational context so a downstream
+                    // proof failure (corrupted merk, invariant
+                    // violation in the prover, etc.) is identifiable
+                    // as a count-offset-specific failure rather than
+                    // an opaque `MerkError`. Mirrors the
+                    // `prove_aggregate_sum_on_range` wrapping a few
+                    // hundred lines up.
+                    .map_err(|e| Error::CorruptedData(format!(
+                        "prove_count_offset_on_range failed: {}",
+                        e
+                    )))
+            );
+            let mut serialized = Vec::with_capacity(128);
+            encode_into(prove_result.ops.iter(), &mut serialized);
+            // Apply consumed limit slots to the outer accounting.
+            if let Some(outer_limit) = overall_limit.as_mut() {
+                let returned_u16: u16 = prove_result.returned.min(u16::MAX as u64) as u16;
+                *outer_limit = outer_limit.saturating_sub(returned_u16);
+            }
+            return Ok(LayerProof {
+                merk_proof: ProofBytes::Merk(serialized),
+                lower_layers: BTreeMap::new(),
+            })
+            .wrap_with_cost(cost);
+        }
+
         // Whether the surrounding query is an aggregate-count carrier:
         // empty trees that match a `subquery_path` step still need a
         // lower-layer descent so the aggregate-count short-circuit can