Implement suggest_scan_ranges and updated chain validation.

Cargo.toml

@@ -19,6 +19,6 @@ panic = 'abort'
 codegen-units = 1
 
 [patch.crates-io]
-incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "67111e29403c33f2e36d6924167f1d5f04ad0fc2" }
-shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "67111e29403c33f2e36d6924167f1d5f04ad0fc2" }
+incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "bae25ad89c0c192bee625252d2d419bd56638e48" }
+shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "bae25ad89c0c192bee625252d2d419bd56638e48" }
 orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" }

zcash_client_backend/CHANGELOG.md

@@ -17,8 +17,9 @@ and this library adheres to Rust's notion of
   - `ShieldedProtocol`
   - `WalletCommitmentTrees`
   - `WalletRead::{block_metadata, block_fully_scanned, suggest_scan_ranges}`
-  - `WalletWrite::put_blocks`
+  - `WalletWrite::{put_blocks, update_chain_tip}`
   - `chain::CommitmentTreeRoot`
+  - `scanning` A new module containing types required for `suggest_scan_ranges`
   - `testing::MockWalletDb::new`
   - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}`:
 - `zcash_client_backend::wallet::WalletSaplingOutput::note_commitment_tree_position`
@@ -36,8 +37,10 @@ and this library adheres to Rust's notion of
     and its signature has changed; it now subsumes the removed `WalletRead::get_all_nullifiers`.
   - `WalletRead::get_target_and_anchor_heights` now takes its argument as a `NonZeroU32`
   - `chain::scan_cached_blocks` now takes a `from_height` argument that
-    permits the caller to control the starting position of the scan range
-    In addition, the `limit` parameter is now required.
+    permits the caller to control the starting position of the scan range.
+    In addition, the `limit` parameter is now required and has type `usize`.
+  - `chain::BlockSource::with_blocks` now takes its limit as an `Option<usize>`
+    instead of `Option<u32>`.
   - A new `CommitmentTree` variant has been added to `data_api::error::Error`
   - `data_api::wallet::{create_spend_to_address, create_proposed_transaction,
     shield_transparent_funds}` all now require that `WalletCommitmentTrees` be
@@ -67,7 +70,8 @@ and this library adheres to Rust's notion of
   method now takes an optional `BlockMetadata` argument instead of a base commitment
   tree and incremental witnesses for each previously-known note. In addition, the
   return type has now been updated to return a `Result<ScannedBlock, ScanError>`.
-
+- `proto/service.proto` has been updated to include the new GRPC endpoints
+  supported by lightwalletd v0.4.15 
 
 ### Removed
 - `zcash_client_backend::data_api`:
@@ -81,8 +85,8 @@ and this library adheres to Rust's notion of
     feature flag, has been modified by the addition of a `sapling_tree` property. 
   - `wallet::input_selection`:
     - `Proposal::target_height` (use `Proposal::min_target_height` instead).
-- `zcash_client_backend::data_api::chain::validate_chain` TODO: document how
-  to handle validation given out-of-order blocks.
+- `zcash_client_backend::data_api::chain::validate_chain` (logic merged into
+  `chain::scan_cached_blocks`.
 - `zcash_client_backend::data_api::chain::error::{ChainError, Cause}` have been 
   replaced by `zcash_client_backend::scanning::ScanError`
 - `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}`

zcash_client_backend/proto/service.proto

@@ -118,6 +118,22 @@ message TreeState {
     string orchardTree = 6; // orchard commitment tree state
 }
 
+enum ShieldedProtocol {
+    sapling = 0;
+    orchard = 1;
+}
+
+message GetSubtreeRootsArg {
+    uint32 startIndex = 1;                  // Index identifying where to start returning subtree roots
+    ShieldedProtocol shieldedProtocol = 2;  // Shielded protocol to return subtree roots for
+    uint32 maxEntries = 3;                  // Maximum number of entries to return, or 0 for all entries.
+}
+message SubtreeRoot {
+    bytes rootHash = 2;                     // The 32-byte Merkle root of the subtree.
+    bytes completingBlockHash = 3;          // The hash of the block that completed this subtree.
+    uint64 completingBlockHeight = 4;       // The height of the block that completed this subtree in the main chain.
+}
+
 // Results are sorted by height, which makes it easy to issue another
 // request that picks up from where the previous left off.
 message GetAddressUtxosArg {
@@ -142,8 +158,12 @@ service CompactTxStreamer {
     rpc GetLatestBlock(ChainSpec) returns (BlockID) {}
     // Return the compact block corresponding to the given block identifier
     rpc GetBlock(BlockID) returns (CompactBlock) {}
+    // Same as GetBlock except actions contain only nullifiers
+    rpc GetBlockNullifiers(BlockID) returns (CompactBlock) {}
     // Return a list of consecutive compact blocks
     rpc GetBlockRange(BlockRange) returns (stream CompactBlock) {}
+    // Same as GetBlockRange except actions contain only nullifiers
+    rpc GetBlockRangeNullifiers(BlockRange) returns (stream CompactBlock) {}
 
     // Return the requested full (not compact) transaction (as from zcashd)
     rpc GetTransaction(TxFilter) returns (RawTransaction) {}
@@ -177,6 +197,10 @@ service CompactTxStreamer {
     rpc GetTreeState(BlockID) returns (TreeState) {}
     rpc GetLatestTreeState(Empty) returns (TreeState) {}
 
+    // Returns a stream of information about roots of subtrees of the Sapling and Orchard
+    // note commitment trees.
+    rpc GetSubtreeRoots(GetSubtreeRootsArg) returns (stream SubtreeRoot) {}
+
     rpc GetAddressUtxos(GetAddressUtxosArg) returns (GetAddressUtxosReplyList) {}
     rpc GetAddressUtxosStream(GetAddressUtxosArg) returns (stream GetAddressUtxosReply) {}
 

zcash_client_backend/src/data_api.rs

@@ -1,9 +1,9 @@
 //! Interfaces for wallet data persistence & low-level wallet utilities.
 
+use std::cmp;
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::num::NonZeroU32;
-use std::{cmp, ops::Range};
 
 use incrementalmerkletree::Retention;
 use secrecy::SecretVec;
@@ -29,9 +29,11 @@ use crate::{
 };
 
 use self::chain::CommitmentTreeRoot;
+use self::scanning::ScanRange;
 
 pub mod chain;
 pub mod error;
+pub mod scanning;
 pub mod wallet;
 
 pub const SAPLING_SHARD_HEIGHT: u8 = sapling::NOTE_COMMITMENT_TREE_DEPTH / 2;
@@ -87,12 +89,10 @@ pub trait WalletRead {
     /// tree size information for each block; or else the scan is likely to fail if notes belonging
     /// to the wallet are detected.
     ///
+    /// The returned range(s) may include block heights beyond the current chain tip.
+    ///
     /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock
-    fn suggest_scan_ranges(
-        &self,
-        batch_size: usize,
-        limit: usize,
-    ) -> Result<Vec<Range<BlockHeight>>, Self::Error>;
+    fn suggest_scan_ranges(&self) -> Result<Vec<ScanRange>, Self::Error>;
 
     /// Returns the default target height (for the block in which a new
     /// transaction would be mined) and anchor height (to use for a new
@@ -501,6 +501,15 @@ pub trait WalletWrite: WalletRead {
         block: Vec<ScannedBlock<sapling::Nullifier>>,
     ) -> Result<Vec<Self::NoteRef>, Self::Error>;
 
+    /// Updates the wallet's view of the blockchain.
+    ///
+    /// This method is used to provide the wallet with information about the state of the
+    /// blockchain, and detect any previously scanned data that needs to be re-validated
+    /// before proceeding with scanning. It should be called at wallet startup prior to calling
+    /// [`WalletRead::suggest_scan_ranges`] in order to provide the wallet with the information it
+    /// needs to correctly prioritize scanning operations.
+    fn update_chain_tip(&mut self, tip_height: BlockHeight) -> Result<(), Self::Error>;
+
     /// Caches a decrypted transaction in the persistent wallet store.
     fn store_decrypted_tx(
         &mut self,
@@ -569,7 +578,7 @@ pub mod testing {
     use incrementalmerkletree::Address;
     use secrecy::{ExposeSecret, SecretVec};
     use shardtree::{memory::MemoryShardStore, ShardTree, ShardTreeError};
-    use std::{collections::HashMap, convert::Infallible, ops::Range};
+    use std::{collections::HashMap, convert::Infallible};
 
     use zcash_primitives::{
         block::BlockHash,
@@ -591,9 +600,9 @@ pub mod testing {
     };
 
     use super::{
-        chain::CommitmentTreeRoot, BlockMetadata, DecryptedTransaction, NullifierQuery,
-        ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, WalletWrite,
-        SAPLING_SHARD_HEIGHT,
+        chain::CommitmentTreeRoot, scanning::ScanRange, BlockMetadata, DecryptedTransaction,
+        NullifierQuery, ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead,
+        WalletWrite, SAPLING_SHARD_HEIGHT,
     };
 
     pub struct MockWalletDb {
@@ -634,11 +643,7 @@ pub mod testing {
             Ok(None)
         }
 
-        fn suggest_scan_ranges(
-            &self,
-            _batch_size: usize,
-            _limit: usize,
-        ) -> Result<Vec<Range<BlockHeight>>, Self::Error> {
+        fn suggest_scan_ranges(&self) -> Result<Vec<ScanRange>, Self::Error> {
             Ok(vec![])
         }
 
@@ -780,6 +785,10 @@ pub mod testing {
             Ok(vec![])
         }
 
+        fn update_chain_tip(&mut self, _tip_height: BlockHeight) -> Result<(), Self::Error> {
+            Ok(())
+        }
+
         fn store_decrypted_tx(
             &mut self,
             _received_tx: DecryptedTransaction,

zcash_client_backend/src/data_api/chain.rs

@@ -7,18 +7,21 @@
 //! # #[cfg(feature = "test-dependencies")]
 //! # {
 //! use zcash_primitives::{
-//!     consensus::{BlockHeight, Network, Parameters}
+//!     consensus::{BlockHeight, Network, Parameters},
+//!     sapling
 //! };
 //!
 //! use zcash_client_backend::{
 //!     data_api::{
-//!         WalletRead, WalletWrite,
+//!         WalletRead, WalletWrite, WalletCommitmentTrees,
 //!         chain::{
 //!             BlockSource,
+//!             CommitmentTreeRoot,
 //!             error::Error,
 //!             scan_cached_blocks,
 //!             testing as chain_testing,
 //!         },
+//!         scanning::ScanPriority,
 //!         testing,
 //!     },
 //! };
@@ -32,20 +35,111 @@
 //! # fn test() -> Result<(), Error<(), Infallible>> {
 //! let network = Network::TestNetwork;
 //! let block_source = chain_testing::MockBlockSource;
-//! let mut db_data = testing::MockWalletDb::new(Network::TestNetwork);
+//! let mut wallet_db = testing::MockWalletDb::new(Network::TestNetwork);
 //!
-//! // 1) Download new CompactBlocks into block_source.
-//! //
-//! // 2) FIXME: Obtain necessary block metadata for continuity checking?
-//! //
-//! // 3) Scan cached blocks.
-//! //
-//! // FIXME: update documentation on how to detect when a rewind is required.
-//! //
-//! // At this point, the cache and scanned data are locally consistent (though not
-//! // necessarily consistent with the latest chain tip - this would be discovered the
-//! // next time this codepath is executed after new blocks are received).
-//! scan_cached_blocks(&network, &block_source, &mut db_data, BlockHeight::from(0), 10)
+//! // 1) Download note commitment tree data from lightwalletd
+//! let roots: Vec<CommitmentTreeRoot<sapling::Node>> = unimplemented!();
+//!
+//! // 2) Pass the commitment tree data to the database.
+//! wallet_db.put_sapling_subtree_roots(0, &roots).unwrap();
+//!
+//! // 3) Download chain tip metadata from lightwalletd
+//! let tip_height: BlockHeight = unimplemented!();
+//!
+//! // 4) Notify the wallet of the updated chain tip.
+//! wallet_db.update_chain_tip(tip_height).map_err(Error::Wallet)?;
+//!
+//! // 5) Get the suggested scan ranges from the wallet database
+//! let mut scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?;
+//!
+//! // 6) Run the following loop until the wallet's view of the chain tip as of the previous wallet
+//! //    session is valid.
+//! loop {
+//!     // If there is a range of blocks that needs to be verified, it will always be returned as
+//!     // the first element of the vector of suggested ranges.
+//!     match scan_ranges.first() {
+//!         Some(scan_range) if scan_range.priority() == ScanPriority::Verify => {
+//!             // Download the blocks in `scan_range` into the block source, overwriting any
+//!             // existing blocks in this range.
+//!             unimplemented!();
+//!
+//!             // Scan the downloaded blocks
+//!             let scan_result = scan_cached_blocks(
+//!                 &network,
+//!                 &block_source,
+//!                 &mut wallet_db,
+//!                 scan_range.block_range().start,
+//!                 scan_range.len()
+//!             );
+//!
+//!             // Check for scanning errors that indicate that the wallet's chain tip is out of
+//!             // sync with blockchain history.
+//!             match scan_result {
+//!                 Ok(_) => {
+//!                     // At this point, the cache and scanned data are locally consistent (though
+//!                     // not necessarily consistent with the latest chain tip - this would be
+//!                     // discovered the next time this codepath is executed after new blocks are
+//!                     // received) so we can break out of the loop.
+//!                     break;
+//!                 }
+//!                 Err(Error::Scan(err)) if err.is_continuity_error() => {
+//!                     // Pick a height to rewind to, which must be at least one block before
+//!                     // the height at which the error occurred, but may be an earlier height
+//!                     // determined based on heuristics such as the platform, available bandwidth,
+//!                     // size of recent CompactBlocks, etc.
+//!                     let rewind_height = err.at_height().saturating_sub(10);
+//!
+//!                     // Rewind to the chosen height.
+//!                     wallet_db.truncate_to_height(rewind_height).map_err(Error::Wallet)?;
+//!
+//!                     // Delete cached blocks from rewind_height onwards.
+//!                     //
+//!                     // This does imply that assumed-valid blocks will be re-downloaded, but it
+//!                     // is also possible that in the intervening time, a chain reorg has
+//!                     // occurred that orphaned some of those blocks.
+//!                     unimplemented!();
+//!                 }
+//!                 Err(other) => {
+//!                     // Handle or return other errors
+//!                 }
+//!             }
+//!
+//!             // In case we updated the suggested scan ranges, now re-request.
+//!             scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?;
+//!         }
+//!         _ => {
+//!             // Nothing to verify; break out of the loop
+//!             break;
+//!         }
+//!     }
+//! }
+//!
+//! // 7) Loop over the remaining suggested scan ranges, retrieving the requested data and calling
+//! //    `scan_cached_blocks` on each range. Periodically, or if a continuity error is
+//! //    encountered, this process should be repeated starting at step (3).
+//! let scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?;
+//! for scan_range in scan_ranges {
+//!     // Download the blocks in `scan_range` into the block source. While in this example this
+//!     // step is performed in-line, it's fine for the download of scan ranges to be asynchronous
+//!     // and for the scanner to process the downloaded ranges as they become available in a
+//!     // separate thread. The scan ranges should also be broken down into smaller chunks as
+//!     // appropriate, and for ranges with priority `Historic` it can be useful to download and
+//!     // scan the range in reverse order (to discover more recent unspent notes sooner), or from
+//!     // the start and end of the range inwards.
+//!     unimplemented!();
+//!
+//!     // Scan the downloaded blocks.
+//!     let scan_result = scan_cached_blocks(
+//!         &network,
+//!         &block_source,
+//!         &mut wallet_db,
+//!         scan_range.block_range().start,
+//!         scan_range.len()
+//!     )?;
+//!
+//!     // Handle scan errors, etc.
+//! }
+//! # Ok(())
 //! # }
 //! # }
 //! ```
@@ -58,14 +152,12 @@ use zcash_primitives::{
 };
 
 use crate::{
-    data_api::{NullifierQuery, WalletWrite},
+    data_api::{BlockMetadata, NullifierQuery, WalletWrite},
     proto::compact_formats::CompactBlock,
     scan::BatchRunner,
     scanning::{add_block_to_runner, check_continuity, scan_block_with_runner},
 };
 
-use super::BlockMetadata;
-
 pub mod error;
 use error::Error;
 
@@ -114,7 +206,7 @@ pub trait BlockSource {
     fn with_blocks<F, WalletErrT>(
         &self,
         from_height: Option<BlockHeight>,
-        limit: Option<u32>,
+        limit: Option<usize>,
         with_row: F,
     ) -> Result<(), error::Error<WalletErrT, Self::Error>>
     where
@@ -145,7 +237,7 @@ pub fn scan_cached_blocks<ParamsT, DbT, BlockSourceT>(
     block_source: &BlockSourceT,
     data_db: &mut DbT,
     from_height: BlockHeight,
-    limit: u32,
+    limit: usize,
 ) -> Result<(), Error<DbT::Error, BlockSourceT::Error>>
 where
     ParamsT: consensus::Parameters + Send + 'static,
@@ -292,7 +384,7 @@ pub mod testing {
         fn with_blocks<F, DbErrT>(
             &self,
             _from_height: Option<BlockHeight>,
-            _limit: Option<u32>,
+            _limit: Option<usize>,
             _with_row: F,
         ) -> Result<(), Error<DbErrT, Infallible>>
         where