From f1a2b95018f8e005f62d3f8aba52ffe0dae98562 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 May 2023 05:57:59 +0000 Subject: [PATCH 0001/1122] Bump codecov/codecov-action from 3.1.3 to 3.1.4 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3.1.3 to 3.1.4. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3.1.3...v3.1.4) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0611a39c6c..36ac21b7ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,7 +121,7 @@ jobs: - name: Generate coverage report run: cargo tarpaulin --engine llvm --all-features --release --timeout 600 --out Xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3.1.3 + uses: codecov/codecov-action@v3.1.4 doc-links: name: Intra-doc links From 8d86ffd9c421f60c4c0f3f59d54cf2e9d957c27b Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 6 Jun 2023 12:35:39 -0600 Subject: [PATCH 0002/1122] zcash_client_sqlite: Use `prepare_cached` instead of manual statement caching. `rusqlite` includes a mechanism for creating prepared statements that automatically caches them and reuses the caches when possible. This means that it's unnecessary for us to do our own caching, and also offers a minor performance improvement in that we don't need to eagerly prepare statements that we may not execute in the lifetime of a given `WalletDb` object. It also improves code locality, because the prepared statements are now adjacent in the code to the parameter assignment blocks that correspond to those statements. This also updates a number of `put_x` methods to use sqlite upsert functionality via the `ON CONFLICT` clause, instead of having to perform separate inserts and updates. --- zcash_client_sqlite/src/prepared.rs | 147 ---------------------------- zcash_client_sqlite/src/wallet.rs | 77 +++++++++++---- 2 files changed, 59 insertions(+), 165 deletions(-) diff --git a/zcash_client_sqlite/src/prepared.rs b/zcash_client_sqlite/src/prepared.rs index da97faa6cb..92731d82d5 100644 --- a/zcash_client_sqlite/src/prepared.rs +++ b/zcash_client_sqlite/src/prepared.rs @@ -89,14 +89,6 @@ impl<'a> InsertAddress<'a> { /// [`WalletWrite`]: zcash_client_backend::data_api::WalletWrite pub struct DataConnStmtCache<'a, P> { pub(crate) wallet_db: &'a WalletDb

, - stmt_insert_block: Statement<'a>, - - stmt_insert_tx_meta: Statement<'a>, - stmt_update_tx_meta: Statement<'a>, - - stmt_insert_tx_data: Statement<'a>, - stmt_update_tx_data: Statement<'a>, - stmt_select_tx_ref: Statement<'a>, stmt_mark_sapling_note_spent: Statement<'a>, #[cfg(feature = "transparent-inputs")] @@ -129,32 +121,6 @@ impl<'a, P> DataConnStmtCache<'a, P> { Ok( DataConnStmtCache { wallet_db, - stmt_insert_block: wallet_db.conn.prepare( - "INSERT INTO blocks (height, hash, time, sapling_tree) - VALUES (?, ?, ?, ?)", - )?, - stmt_insert_tx_meta: wallet_db.conn.prepare( - "INSERT INTO transactions (txid, block, tx_index) - VALUES (?, ?, ?)", - )?, - stmt_update_tx_meta: wallet_db.conn.prepare( - "UPDATE transactions - SET block = ?, tx_index = ? WHERE txid = ?", - )?, - stmt_insert_tx_data: wallet_db.conn.prepare( - "INSERT INTO transactions (txid, created, expiry_height, raw, fee) - VALUES (?, ?, ?, ?, ?)", - )?, - stmt_update_tx_data: wallet_db.conn.prepare( - "UPDATE transactions - SET expiry_height = :expiry_height, - raw = :raw, - fee = IFNULL(:fee, fee) - WHERE txid = :txid", - )?, - stmt_select_tx_ref: wallet_db.conn.prepare( - "SELECT id_tx FROM transactions WHERE txid = ?", - )?, stmt_mark_sapling_note_spent: wallet_db.conn.prepare( "UPDATE sapling_received_notes SET spent = ? WHERE nf = ?" )?, @@ -271,119 +237,6 @@ impl<'a, P> DataConnStmtCache<'a, P> { ) } - /// Inserts information about a scanned block into the database. - pub fn stmt_insert_block( - &mut self, - block_height: BlockHeight, - block_hash: BlockHash, - block_time: u32, - commitment_tree: &sapling::CommitmentTree, - ) -> Result<(), SqliteClientError> { - let mut encoded_tree = Vec::new(); - write_commitment_tree(commitment_tree, &mut encoded_tree).unwrap(); - - self.stmt_insert_block.execute(params![ - u32::from(block_height), - &block_hash.0[..], - block_time, - encoded_tree - ])?; - - Ok(()) - } - - /// Inserts the given transaction and its block metadata into the wallet. - /// - /// Returns the database row for the newly-inserted transaction, or an error if the - /// transaction exists. - pub(crate) fn stmt_insert_tx_meta( - &mut self, - txid: &TxId, - height: BlockHeight, - tx_index: usize, - ) -> Result { - self.stmt_insert_tx_meta.execute(params![ - &txid.as_ref()[..], - u32::from(height), - (tx_index as i64), - ])?; - - Ok(self.wallet_db.conn.last_insert_rowid()) - } - - /// Updates the block metadata for the given transaction. - /// - /// Returns `false` if the transaction doesn't exist in the wallet. - pub(crate) fn stmt_update_tx_meta( - &mut self, - height: BlockHeight, - tx_index: usize, - txid: &TxId, - ) -> Result { - match self.stmt_update_tx_meta.execute(params![ - u32::from(height), - (tx_index as i64), - &txid.as_ref()[..], - ])? { - 0 => Ok(false), - 1 => Ok(true), - _ => unreachable!("txid column is marked as UNIQUE"), - } - } - - /// Inserts the given transaction and its data into the wallet. - /// - /// Returns the database row for the newly-inserted transaction, or an error if the - /// transaction exists. - pub(crate) fn stmt_insert_tx_data( - &mut self, - txid: &TxId, - created_at: Option, - expiry_height: BlockHeight, - raw_tx: &[u8], - fee: Option, - ) -> Result { - self.stmt_insert_tx_data.execute(params![ - &txid.as_ref()[..], - created_at, - u32::from(expiry_height), - raw_tx, - fee.map(i64::from) - ])?; - - Ok(self.wallet_db.conn.last_insert_rowid()) - } - - /// Updates the data for the given transaction. - /// - /// Returns `false` if the transaction doesn't exist in the wallet. - pub(crate) fn stmt_update_tx_data( - &mut self, - expiry_height: BlockHeight, - raw_tx: &[u8], - fee: Option, - txid: &TxId, - ) -> Result { - let sql_args: &[(&str, &dyn ToSql)] = &[ - (":expiry_height", &u32::from(expiry_height)), - (":raw", &raw_tx), - (":fee", &fee.map(i64::from)), - (":txid", &&txid.as_ref()[..]), - ]; - match self.stmt_update_tx_data.execute(sql_args)? { - 0 => Ok(false), - 1 => Ok(true), - _ => unreachable!("txid column is marked as UNIQUE"), - } - } - - /// Finds the database row for the given `txid`, if the transaction is in the wallet. - pub(crate) fn stmt_select_tx_ref(&mut self, txid: &TxId) -> Result { - self.stmt_select_tx_ref - .query_row([&txid.as_ref()[..]], |row| row.get(0)) - .map_err(SqliteClientError::from) - } - /// Marks a given nullifier as having been revealed in the construction of the /// specified transaction. /// diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index dbffcaadd2..1d9d897f47 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -64,7 +64,7 @@ //! wallet. //! - `memo` the shielded memo associated with the output, if any. -use rusqlite::{named_params, OptionalExtension, ToSql}; +use rusqlite::{named_params, params, OptionalExtension, ToSql}; use std::collections::HashMap; use std::convert::TryFrom; @@ -72,6 +72,7 @@ use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight, BranchId, NetworkUpgrade, Parameters}, memo::{Memo, MemoBytes}, + merkle_tree::{write_commitment_tree, write_incremental_witness}, sapling::CommitmentTree, transaction::{components::Amount, Transaction, TxId}, zip32::{ @@ -94,7 +95,7 @@ use crate::{ #[cfg(feature = "transparent-inputs")] use { crate::UtxoId, - rusqlite::{params, Connection}, + rusqlite::Connection, std::collections::BTreeSet, zcash_client_backend::{ address::AddressMetadata, encoding::AddressCodec, wallet::WalletTransparentOutput, @@ -739,7 +740,22 @@ pub(crate) fn insert_block<'a, P>( block_time: u32, commitment_tree: &CommitmentTree, ) -> Result<(), SqliteClientError> { - stmts.stmt_insert_block(block_height, block_hash, block_time, commitment_tree) + let mut encoded_tree = Vec::new(); + write_commitment_tree(commitment_tree, &mut encoded_tree).unwrap(); + + let mut stmt_insert_block = stmts.wallet_db.conn.prepare_cached( + "INSERT INTO blocks (height, hash, time, sapling_tree) + VALUES (?, ?, ?, ?)", + )?; + + stmt_insert_block.execute(params![ + u32::from(block_height), + &block_hash.0[..], + block_time, + encoded_tree + ])?; + + Ok(()) } /// Inserts information about a mined transaction that was observed to @@ -749,13 +765,25 @@ pub(crate) fn put_tx_meta<'a, P, N>( tx: &WalletTx, height: BlockHeight, ) -> Result { - if !stmts.stmt_update_tx_meta(height, tx.index, &tx.txid)? { - // It isn't there, so insert our transaction into the database. - stmts.stmt_insert_tx_meta(&tx.txid, height, tx.index) - } else { - // It was there, so grab its row number. - stmts.stmt_select_tx_ref(&tx.txid) - } + // It isn't there, so insert our transaction into the database. + let mut stmt_upsert_tx_meta = stmts.wallet_db.conn.prepare_cached( + "INSERT INTO transactions (txid, block, tx_index) + VALUES (:txid, :block, :tx_index) + ON CONFLICT (txid) DO UPDATE + SET block = :block, + tx_index = :tx_index + RETURNING id_tx", + )?; + + let tx_params = named_params![ + ":txid": &tx.txid.as_ref()[..], + ":block": u32::from(height), + ":tx_index": i64::try_from(tx.index).expect("transaction indices are representable as i64"), + ]; + + stmt_upsert_tx_meta + .query_row(tx_params, |row| row.get::<_, i64>(0)) + .map_err(SqliteClientError::from) } /// Inserts full transaction data into the database. @@ -765,18 +793,31 @@ pub(crate) fn put_tx_data<'a, P>( fee: Option, created_at: Option, ) -> Result { - let txid = tx.txid(); + let mut stmt_upsert_tx_data = stmts.wallet_db.conn.prepare_cached( + "INSERT INTO transactions (txid, created, expiry_height, raw, fee) + VALUES (:txid, :created_at, :expiry_height, :raw, :fee) + ON CONFLICT (txid) DO UPDATE + SET expiry_height = :expiry_height, + raw = :raw, + fee = IFNULL(:fee, fee) + RETURNING id_tx", + )?; + let txid = tx.txid(); let mut raw_tx = vec![]; tx.write(&mut raw_tx)?; - if !stmts.stmt_update_tx_data(tx.expiry_height(), &raw_tx, fee, &txid)? { - // It isn't there, so insert our transaction into the database. - stmts.stmt_insert_tx_data(&txid, created_at, tx.expiry_height(), &raw_tx, fee) - } else { - // It was there, so grab its row number. - stmts.stmt_select_tx_ref(&txid) - } + let tx_params = named_params![ + ":txid": &txid.as_ref()[..], + ":created_at": created_at, + ":expiry_height": u32::from(tx.expiry_height()), + ":raw": raw_tx, + ":fee": fee.map(i64::from), + ]; + + stmt_upsert_tx_data + .query_row(tx_params, |row| row.get::<_, i64>(0)) + .map_err(SqliteClientError::from) } /// Marks the given UTXO as having been spent. From 7917effe82eaaf6fc40317a9f90ead894befa3da Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 7 Jun 2023 15:03:20 -0600 Subject: [PATCH 0003/1122] zcash_client_sqlite: Use upsert & automatic caching of prepared statements for `put_received_note` --- zcash_client_sqlite/src/lib.rs | 29 +++-- zcash_client_sqlite/src/prepared.rs | 148 ---------------------- zcash_client_sqlite/src/wallet.rs | 2 +- zcash_client_sqlite/src/wallet/sapling.rs | 115 +++++++++-------- 4 files changed, 81 insertions(+), 213 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 60cc142ae0..54c55ac533 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -43,7 +43,7 @@ use zcash_primitives::{ consensus::{self, BlockHeight}, legacy::TransparentAddress, memo::{Memo, MemoBytes}, - sapling::{self}, + sapling, transaction::{ components::{amount::Amount, OutPoint}, Transaction, TxId, @@ -218,7 +218,7 @@ impl WalletRead for WalletDb

{ ) -> Result, Self::Error> { match query { NullifierQuery::Unspent => wallet::sapling::get_sapling_nullifiers(self), - NullifierQuery::All => wallet::sapling::get_all_sapling_nullifiers(self), + NullifierQuery::All => wallet::sapling::get_all_sapling_nullifiers(&self.conn), } } @@ -228,7 +228,7 @@ impl WalletRead for WalletDb

{ anchor_height: BlockHeight, exclude: &[Self::NoteRef], ) -> Result>, Self::Error> { - wallet::sapling::get_spendable_sapling_notes(self, account, anchor_height, exclude) + wallet::sapling::get_spendable_sapling_notes(&self.conn, account, anchor_height, exclude) } fn select_spendable_sapling_notes( @@ -239,7 +239,7 @@ impl WalletRead for WalletDb

{ exclude: &[Self::NoteRef], ) -> Result>, Self::Error> { wallet::sapling::select_spendable_sapling_notes( - self, + &self.conn, account, target_value, anchor_height, @@ -534,11 +534,16 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { // Mark notes as spent and remove them from the scanning cache for spend in &tx.sapling_spends { - wallet::sapling::mark_sapling_note_spent(up, tx_row, spend.nf())?; + wallet::sapling::mark_sapling_note_spent( + &up.wallet_db.conn, + tx_row, + spend.nf(), + )?; } for output in &tx.sapling_outputs { - let received_note_id = wallet::sapling::put_received_note(up, output, tx_row)?; + let received_note_id = + wallet::sapling::put_received_note(&up.wallet_db.conn, output, tx_row)?; // Save witness for note. new_witnesses.push((received_note_id, output.witness().clone())); @@ -594,7 +599,7 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { )?; if matches!(recipient, Recipient::InternalAccount(_, _)) { - wallet::sapling::put_received_note(up, output, tx_ref)?; + wallet::sapling::put_received_note(&up.wallet_db.conn, output, tx_ref)?; } } TransferType::Incoming => { @@ -608,7 +613,7 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { } } - wallet::sapling::put_received_note(up, output, tx_ref)?; + wallet::sapling::put_received_note(&up.wallet_db.conn, output, tx_ref)?; } } } @@ -669,7 +674,11 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { // reasonable assumption for a light client such as a mobile phone. if let Some(bundle) = sent_tx.tx.sapling_bundle() { for spend in bundle.shielded_spends() { - wallet::sapling::mark_sapling_note_spent(up, tx_ref, spend.nullifier())?; + wallet::sapling::mark_sapling_note_spent( + &up.wallet_db.conn, + tx_ref, + spend.nullifier(), + )?; } } @@ -683,7 +692,7 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { if let Some((account, note)) = output.sapling_change_to() { wallet::sapling::put_received_note( - up, + &up.wallet_db.conn, &DecryptedOutput { index: output.output_index(), note: note.clone(), diff --git a/zcash_client_sqlite/src/prepared.rs b/zcash_client_sqlite/src/prepared.rs index 92731d82d5..c6d94d364b 100644 --- a/zcash_client_sqlite/src/prepared.rs +++ b/zcash_client_sqlite/src/prepared.rs @@ -90,7 +90,6 @@ impl<'a> InsertAddress<'a> { pub struct DataConnStmtCache<'a, P> { pub(crate) wallet_db: &'a WalletDb

, - stmt_mark_sapling_note_spent: Statement<'a>, #[cfg(feature = "transparent-inputs")] stmt_mark_transparent_utxo_spent: Statement<'a>, @@ -102,9 +101,6 @@ pub struct DataConnStmtCache<'a, P> { stmt_insert_legacy_transparent_utxo: Statement<'a>, #[cfg(feature = "transparent-inputs")] stmt_update_legacy_transparent_utxo: Statement<'a>, - stmt_insert_received_note: Statement<'a>, - stmt_update_received_note: Statement<'a>, - stmt_select_received_note: Statement<'a>, stmt_insert_sent_output: Statement<'a>, stmt_update_sent_output: Statement<'a>, @@ -121,9 +117,6 @@ impl<'a, P> DataConnStmtCache<'a, P> { Ok( DataConnStmtCache { wallet_db, - stmt_mark_sapling_note_spent: wallet_db.conn.prepare( - "UPDATE sapling_received_notes SET spent = ? WHERE nf = ?" - )?, #[cfg(feature = "transparent-inputs")] stmt_mark_transparent_utxo_spent: wallet_db.conn.prepare( "UPDATE utxos SET spent_in_tx = :spent_in_tx @@ -182,24 +175,6 @@ impl<'a, P> DataConnStmtCache<'a, P> { AND prevout_idx = :prevout_idx RETURNING id_utxo" )?, - stmt_insert_received_note: wallet_db.conn.prepare( - "INSERT INTO sapling_received_notes (tx, output_index, account, diversifier, value, rcm, memo, nf, is_change) - VALUES (:tx, :output_index, :account, :diversifier, :value, :rcm, :memo, :nf, :is_change)", - )?, - stmt_update_received_note: wallet_db.conn.prepare( - "UPDATE sapling_received_notes - SET account = :account, - diversifier = :diversifier, - value = :value, - rcm = :rcm, - nf = IFNULL(:nf, nf), - memo = IFNULL(:memo, memo), - is_change = IFNULL(:is_change, is_change) - WHERE tx = :tx AND output_index = :output_index", - )?, - stmt_select_received_note: wallet_db.conn.prepare( - "SELECT id_note FROM sapling_received_notes WHERE tx = ? AND output_index = ?" - )?, stmt_update_sent_output: wallet_db.conn.prepare( "UPDATE sent_notes SET from_account = :from_account, @@ -237,28 +212,6 @@ impl<'a, P> DataConnStmtCache<'a, P> { ) } - /// Marks a given nullifier as having been revealed in the construction of the - /// specified transaction. - /// - /// Marking a note spent in this fashion does NOT imply that the spending transaction - /// has been mined. - /// - /// Returns `false` if the nullifier does not correspond to any received note. - pub(crate) fn stmt_mark_sapling_note_spent( - &mut self, - tx_ref: i64, - nf: &Nullifier, - ) -> Result { - match self - .stmt_mark_sapling_note_spent - .execute(params![tx_ref, &nf.0[..]])? - { - 0 => Ok(false), - 1 => Ok(true), - _ => unreachable!("nf column is marked as UNIQUE"), - } - } - /// Marks the given UTXO as having been spent. /// /// Returns `false` if `outpoint` does not correspond to any tracked UTXO. @@ -520,107 +473,6 @@ impl<'a, P: consensus::Parameters> DataConnStmtCache<'a, P> { } impl<'a, P> DataConnStmtCache<'a, P> { - /// Inserts the given received note into the wallet. - /// - /// This implementation relies on the facts that: - /// - A transaction will not contain more than 2^63 shielded outputs. - /// - A note value will never exceed 2^63 zatoshis. - /// - /// Returns the database row for the newly-inserted note, or an error if the note - /// exists. - #[allow(clippy::too_many_arguments)] - pub(crate) fn stmt_insert_received_note( - &mut self, - tx_ref: i64, - output_index: usize, - account: AccountId, - diversifier: &Diversifier, - value: u64, - rcm: [u8; 32], - nf: Option<&Nullifier>, - memo: Option<&MemoBytes>, - is_change: bool, - ) -> Result { - let sql_args: &[(&str, &dyn ToSql)] = &[ - (":tx", &tx_ref), - (":output_index", &(output_index as i64)), - (":account", &u32::from(account)), - (":diversifier", &diversifier.0.as_ref()), - (":value", &(value as i64)), - (":rcm", &rcm.as_ref()), - (":nf", &nf.map(|nf| nf.0.as_ref())), - ( - ":memo", - &memo - .filter(|m| *m != &MemoBytes::empty()) - .map(|m| m.as_slice()), - ), - (":is_change", &is_change), - ]; - - self.stmt_insert_received_note.execute(sql_args)?; - - Ok(NoteId::ReceivedNoteId( - self.wallet_db.conn.last_insert_rowid(), - )) - } - - /// Updates the data for the given transaction. - /// - /// This implementation relies on the facts that: - /// - A transaction will not contain more than 2^63 shielded outputs. - /// - A note value will never exceed 2^63 zatoshis. - /// - /// Returns `false` if the transaction doesn't exist in the wallet. - #[allow(clippy::too_many_arguments)] - pub(crate) fn stmt_update_received_note( - &mut self, - account: AccountId, - diversifier: &Diversifier, - value: u64, - rcm: [u8; 32], - nf: Option<&Nullifier>, - memo: Option<&MemoBytes>, - is_change: bool, - tx_ref: i64, - output_index: usize, - ) -> Result { - let sql_args: &[(&str, &dyn ToSql)] = &[ - (":account", &u32::from(account)), - (":diversifier", &diversifier.0.as_ref()), - (":value", &(value as i64)), - (":rcm", &rcm.as_ref()), - (":nf", &nf.map(|nf| nf.0.as_ref())), - ( - ":memo", - &memo - .filter(|m| *m != &MemoBytes::empty()) - .map(|m| m.as_slice()), - ), - (":is_change", &is_change), - (":tx", &tx_ref), - (":output_index", &(output_index as i64)), - ]; - - match self.stmt_update_received_note.execute(sql_args)? { - 0 => Ok(false), - 1 => Ok(true), - _ => unreachable!("tx_output constraint is marked as UNIQUE"), - } - } - - /// Finds the database row for the given `txid`, if the transaction is in the wallet. - pub(crate) fn stmt_select_received_note( - &mut self, - tx_ref: i64, - output_index: usize, - ) -> Result { - self.stmt_select_received_note - .query_row(params![tx_ref, (output_index as i64)], |row| { - row.get(0).map(NoteId::ReceivedNoteId) - }) - .map_err(SqliteClientError::from) - } /// Records the incremental witness for the specified note, as of the given block /// height. diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 1d9d897f47..00c935a8d0 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -768,7 +768,7 @@ pub(crate) fn put_tx_meta<'a, P, N>( // It isn't there, so insert our transaction into the database. let mut stmt_upsert_tx_meta = stmts.wallet_db.conn.prepare_cached( "INSERT INTO transactions (txid, block, tx_index) - VALUES (:txid, :block, :tx_index) + VALUES (:txid, :block, :tx_index) ON CONFLICT (txid) DO UPDATE SET block = :block, tx_index = :tx_index diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index a18281dc87..9419231baf 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -1,6 +1,6 @@ //! Functions for Sapling support in the wallet. use group::ff::PrimeField; -use rusqlite::{named_params, types::Value, OptionalExtension, Row}; +use rusqlite::{named_params, params, types::Value, Connection, OptionalExtension, Row}; use std::rc::Rc; use zcash_primitives::{ @@ -117,13 +117,13 @@ fn to_spendable_note(row: &Row) -> Result, SqliteCli }) } -pub(crate) fn get_spendable_sapling_notes

( - wdb: &WalletDb

, +pub(crate) fn get_spendable_sapling_notes( + conn: &Connection, account: AccountId, anchor_height: BlockHeight, exclude: &[NoteId], ) -> Result>, SqliteClientError> { - let mut stmt_select_notes = wdb.conn.prepare( + let mut stmt_select_notes = conn.prepare_cached( "SELECT id_note, diversifier, value, rcm, witness FROM sapling_received_notes INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx @@ -156,8 +156,8 @@ pub(crate) fn get_spendable_sapling_notes

( notes.collect::>() } -pub(crate) fn select_spendable_sapling_notes

( - wdb: &WalletDb

, +pub(crate) fn select_spendable_sapling_notes( + conn: &Connection, account: AccountId, target_value: Amount, anchor_height: BlockHeight, @@ -181,7 +181,7 @@ pub(crate) fn select_spendable_sapling_notes

( // required value, bringing the sum of all selected notes across the threshold. // // 4) Match the selected notes against the witnesses at the desired height. - let mut stmt_select_notes = wdb.conn.prepare( + let mut stmt_select_notes = conn.prepare_cached( "WITH selected AS ( WITH eligible AS ( SELECT id_note, diversifier, value, rcm, @@ -189,8 +189,8 @@ pub(crate) fn select_spendable_sapling_notes

( (PARTITION BY account, spent ORDER BY id_note) AS so_far FROM sapling_received_notes INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx - WHERE account = :account - AND spent IS NULL + WHERE account = :account + AND spent IS NULL AND transactions.block <= :anchor_height AND id_note NOT IN rarray(:exclude) ) @@ -318,11 +318,11 @@ pub(crate) fn get_sapling_nullifiers

( } /// Returns the nullifiers for the notes that this wallet is tracking. -pub(crate) fn get_all_sapling_nullifiers

( - wdb: &WalletDb

, +pub(crate) fn get_all_sapling_nullifiers( + conn: &Connection, ) -> Result, SqliteClientError> { // Get the nullifiers for the notes we are tracking - let mut stmt_fetch_nullifiers = wdb.conn.prepare( + let mut stmt_fetch_nullifiers = conn.prepare( "SELECT rn.id_note, rn.account, rn.nf FROM sapling_received_notes rn WHERE nf IS NOT NULL", @@ -345,13 +345,19 @@ pub(crate) fn get_all_sapling_nullifiers

( /// /// Marking a note spent in this fashion does NOT imply that the /// spending transaction has been mined. -pub(crate) fn mark_sapling_note_spent<'a, P>( - stmts: &mut DataConnStmtCache<'a, P>, +pub(crate) fn mark_sapling_note_spent( + conn: &Connection, tx_ref: i64, nf: &Nullifier, -) -> Result<(), SqliteClientError> { - stmts.stmt_mark_sapling_note_spent(tx_ref, nf)?; - Ok(()) +) -> Result { + let mut stmt_mark_sapling_note_spent = + conn.prepare_cached("UPDATE sapling_received_notes SET spent = ? WHERE nf = ?")?; + + match stmt_mark_sapling_note_spent.execute(params![tx_ref, &nf.0[..]])? { + 0 => Ok(false), + 1 => Ok(true), + _ => unreachable!("nf column is marked as UNIQUE"), + } } /// Records the specified shielded output as having been received. @@ -359,49 +365,50 @@ pub(crate) fn mark_sapling_note_spent<'a, P>( /// This implementation relies on the facts that: /// - A transaction will not contain more than 2^63 shielded outputs. /// - A note value will never exceed 2^63 zatoshis. -pub(crate) fn put_received_note<'a, P, T: ReceivedSaplingOutput>( - stmts: &mut DataConnStmtCache<'a, P>, +pub(crate) fn put_received_note( + conn: &Connection, output: &T, tx_ref: i64, ) -> Result { + let mut stmt_upsert_received_note = conn.prepare_cached( + "INSERT INTO sapling_received_notes + (tx, output_index, account, diversifier, value, rcm, memo, nf, is_change) + VALUES + (:tx, :output_index, :account, :diversifier, :value, :rcm, :memo, :nf, :is_change) + ON CONFLICT (tx, output_index) DO UPDATE + SET account = :account, + diversifier = :diversifier, + value = :value, + rcm = :rcm, + nf = IFNULL(:nf, nf), + memo = IFNULL(:memo, memo), + is_change = IFNULL(:is_change, is_change) + RETURNING id_note", + )?; + let rcm = output.note().rcm().to_repr(); - let account = output.account(); let to = output.note().recipient(); let diversifier = to.diversifier(); - let value = output.note().value(); - let memo = output.memo(); - let is_change = output.is_change(); - let output_index = output.index(); - let nf = output.nullifier(); - - // First try updating an existing received note into the database. - if !stmts.stmt_update_received_note( - account, - diversifier, - value.inner(), - rcm, - nf, - memo, - is_change, - tx_ref, - output_index, - )? { - // It isn't there, so insert our note into the database. - stmts.stmt_insert_received_note( - tx_ref, - output_index, - account, - diversifier, - value.inner(), - rcm, - nf, - memo, - is_change, - ) - } else { - // It was there, so grab its row number. - stmts.stmt_select_received_note(tx_ref, output.index()) - } + + let sql_args = named_params![ + ":tx": &tx_ref, + ":output_index": i64::try_from(output.index()).expect("output indices are representable as i64"), + ":account": u32::from(output.account()), + ":diversifier": &diversifier.0.as_ref(), + ":value": output.note().value().inner(), + ":rcm": &rcm.as_ref(), + ":nf": output.nullifier().map(|nf| nf.0.as_ref()), + ":memo": output.memo() + .filter(|m| *m != &MemoBytes::empty()) + .map(|m| m.as_slice()), + ":is_change": output.is_change() + ]; + + stmt_upsert_received_note + .query_row(sql_args, |row| { + row.get::<_, i64>(0).map(NoteId::ReceivedNoteId) + }) + .map_err(SqliteClientError::from) } #[cfg(test)] From 2354c8b48ddf7d766b5d933056ff9a7696a28c1f Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 7 Jun 2023 15:41:52 -0600 Subject: [PATCH 0004/1122] zcash_client_sqlite: Use upsert & automatic caching of prepared statements for `put_sent_output` --- zcash_client_sqlite/src/lib.rs | 14 ++- zcash_client_sqlite/src/prepared.rs | 137 +--------------------------- zcash_client_sqlite/src/wallet.rs | 103 ++++++++++++++++----- 3 files changed, 96 insertions(+), 158 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 54c55ac533..c2361d29fc 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -588,7 +588,8 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { }; wallet::put_sent_output( - up, + &up.wallet_db.conn, + &up.wallet_db.params, output.account, tx_ref, output.index, @@ -638,7 +639,8 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { for (output_index, txout) in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vout.iter()).enumerate() { if let Some(address) = txout.recipient_address() { wallet::put_sent_output( - up, + &up.wallet_db.conn, + &up.wallet_db.params, *account_id, tx_ref, output_index, @@ -688,7 +690,13 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { } for output in &sent_tx.outputs { - wallet::insert_sent_output(up, tx_ref, sent_tx.account, output)?; + wallet::insert_sent_output( + &up.wallet_db.conn, + &up.wallet_db.params, + tx_ref, + sent_tx.account, + output, + )?; if let Some((account, note)) = output.sapling_change_to() { wallet::sapling::put_received_note( diff --git a/zcash_client_sqlite/src/prepared.rs b/zcash_client_sqlite/src/prepared.rs index c6d94d364b..1f6779931e 100644 --- a/zcash_client_sqlite/src/prepared.rs +++ b/zcash_client_sqlite/src/prepared.rs @@ -9,22 +9,15 @@ use rusqlite::{named_params, params, Statement, ToSql}; use zcash_primitives::{ - block::BlockHash, consensus::{self, BlockHeight}, - memo::MemoBytes, - merkle_tree::{write_commitment_tree, write_incremental_witness}, - sapling::{self, Diversifier, Nullifier}, - transaction::{components::Amount, TxId}, + merkle_tree::write_incremental_witness, + sapling, zip32::{AccountId, DiversifierIndex}, }; -use zcash_client_backend::{ - address::UnifiedAddress, - data_api::{PoolType, Recipient}, - encoding::AddressCodec, -}; +use zcash_client_backend::{address::UnifiedAddress, encoding::AddressCodec}; -use crate::{error::SqliteClientError, wallet::pool_code, NoteId, WalletDb}; +use crate::{error::SqliteClientError, NoteId, WalletDb}; #[cfg(feature = "transparent-inputs")] use { @@ -102,9 +95,6 @@ pub struct DataConnStmtCache<'a, P> { #[cfg(feature = "transparent-inputs")] stmt_update_legacy_transparent_utxo: Statement<'a>, - stmt_insert_sent_output: Statement<'a>, - stmt_update_sent_output: Statement<'a>, - stmt_insert_witness: Statement<'a>, stmt_prune_witnesses: Statement<'a>, stmt_update_expired: Statement<'a>, @@ -175,25 +165,6 @@ impl<'a, P> DataConnStmtCache<'a, P> { AND prevout_idx = :prevout_idx RETURNING id_utxo" )?, - stmt_update_sent_output: wallet_db.conn.prepare( - "UPDATE sent_notes - SET from_account = :from_account, - to_address = :to_address, - to_account = :to_account, - value = :value, - memo = IFNULL(:memo, memo) - WHERE tx = :tx - AND output_pool = :output_pool - AND output_index = :output_index", - )?, - stmt_insert_sent_output: wallet_db.conn.prepare( - "INSERT INTO sent_notes ( - tx, output_pool, output_index, from_account, - to_address, to_account, value, memo) - VALUES ( - :tx, :output_pool, :output_index, :from_account, - :to_address, :to_account, :value, :memo)" - )?, stmt_insert_witness: wallet_db.conn.prepare( "INSERT INTO sapling_witnesses (note, block, witness) VALUES (?, ?, ?)", @@ -236,105 +207,6 @@ impl<'a, P> DataConnStmtCache<'a, P> { } impl<'a, P: consensus::Parameters> DataConnStmtCache<'a, P> { - /// Inserts a sent note into the wallet database. - /// - /// `output_index` is the index within the transaction that contains the recipient output: - /// - /// - If `to` is a Unified address, this is an index into the outputs of the transaction - /// within the bundle associated with the recipient's output pool. - /// - If `to` is a Sapling address, this is an index into the Sapling outputs of the - /// transaction. - /// - If `to` is a transparent address, this is an index into the transparent outputs of - /// the transaction. - /// - If `to` is an internal account, this is an index into the Sapling outputs of the - /// transaction. - #[allow(clippy::too_many_arguments)] - pub(crate) fn stmt_insert_sent_output( - &mut self, - tx_ref: i64, - output_index: usize, - from_account: AccountId, - to: &Recipient, - value: Amount, - memo: Option<&MemoBytes>, - ) -> Result<(), SqliteClientError> { - let (to_address, to_account, pool_type) = match to { - Recipient::Transparent(addr) => ( - Some(addr.encode(&self.wallet_db.params)), - None, - PoolType::Transparent, - ), - Recipient::Sapling(addr) => ( - Some(addr.encode(&self.wallet_db.params)), - None, - PoolType::Sapling, - ), - Recipient::Unified(addr, pool) => { - (Some(addr.encode(&self.wallet_db.params)), None, *pool) - } - Recipient::InternalAccount(id, pool) => (None, Some(u32::from(*id)), *pool), - }; - - self.stmt_insert_sent_output.execute(named_params![ - ":tx": &tx_ref, - ":output_pool": &pool_code(pool_type), - ":output_index": &i64::try_from(output_index).unwrap(), - ":from_account": &u32::from(from_account), - ":to_address": &to_address, - ":to_account": &to_account, - ":value": &i64::from(value), - ":memo": &memo.filter(|m| *m != &MemoBytes::empty()).map(|m| m.as_slice()), - ])?; - - Ok(()) - } - - /// Updates the data for the given sent note. - /// - /// Returns `false` if the transaction doesn't exist in the wallet. - #[allow(clippy::too_many_arguments)] - pub(crate) fn stmt_update_sent_output( - &mut self, - from_account: AccountId, - to: &Recipient, - value: Amount, - memo: Option<&MemoBytes>, - tx_ref: i64, - output_index: usize, - ) -> Result { - let (to_address, to_account, pool_type) = match to { - Recipient::Transparent(addr) => ( - Some(addr.encode(&self.wallet_db.params)), - None, - PoolType::Transparent, - ), - Recipient::Sapling(addr) => ( - Some(addr.encode(&self.wallet_db.params)), - None, - PoolType::Sapling, - ), - Recipient::Unified(addr, pool) => { - (Some(addr.encode(&self.wallet_db.params)), None, *pool) - } - Recipient::InternalAccount(id, pool) => (None, Some(u32::from(*id)), *pool), - }; - - match self.stmt_update_sent_output.execute(named_params![ - ":from_account": &u32::from(from_account), - ":to_address": &to_address, - ":to_account": &to_account, - ":value": &i64::from(value), - ":memo": &memo.filter(|m| *m != &MemoBytes::empty()).map(|m| m.as_slice()), - ":tx": &tx_ref, - ":output_pool": &pool_code(pool_type), - ":output_index": &i64::try_from(output_index).unwrap(), - ])? { - 0 => Ok(false), - 1 => Ok(true), - _ => unreachable!("tx_output constraint is marked as UNIQUE"), - } - } - /// Adds the given received UTXO to the datastore. /// /// Returns the database identifier for the newly-inserted UTXO if the address to which the @@ -473,7 +345,6 @@ impl<'a, P: consensus::Parameters> DataConnStmtCache<'a, P> { } impl<'a, P> DataConnStmtCache<'a, P> { - /// Records the incremental witness for the specified note, as of the given block /// height. /// diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 00c935a8d0..8404145d26 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -72,7 +72,7 @@ use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight, BranchId, NetworkUpgrade, Parameters}, memo::{Memo, MemoBytes}, - merkle_tree::{write_commitment_tree, write_incremental_witness}, + merkle_tree::write_commitment_tree, sapling::CommitmentTree, transaction::{components::Amount, Transaction, TxId}, zip32::{ @@ -891,23 +891,63 @@ pub(crate) fn update_expired_notes

( stmts.stmt_update_expired(height) } +// A utility function for creation of parameters for use in `insert_sent_output` +// and `put_sent_output` +// +// - If `to` is a Unified address, this is an index into the outputs of the transaction +// within the bundle associated with the recipient's output pool. +// - If `to` is a Sapling address, this is an index into the Sapling outputs of the +// transaction. +// - If `to` is a transparent address, this is an index into the transparent outputs of +// the transaction. +// - If `to` is an internal account, this is an index into the Sapling outputs of the +// transaction. +fn recipient_params( + params: &P, + to: &Recipient, +) -> (Option, Option, PoolType) { + match to { + Recipient::Transparent(addr) => (Some(addr.encode(params)), None, PoolType::Transparent), + Recipient::Sapling(addr) => (Some(addr.encode(params)), None, PoolType::Sapling), + Recipient::Unified(addr, pool) => (Some(addr.encode(params)), None, *pool), + Recipient::InternalAccount(id, pool) => (None, Some(u32::from(*id)), *pool), + } +} + /// Records information about a transaction output that your wallet created. /// /// This is a crate-internal convenience method. -pub(crate) fn insert_sent_output<'a, P: consensus::Parameters>( - stmts: &mut DataConnStmtCache<'a, P>, +pub(crate) fn insert_sent_output( + conn: &Connection, + params: &P, tx_ref: i64, from_account: AccountId, output: &SentTransactionOutput, ) -> Result<(), SqliteClientError> { - stmts.stmt_insert_sent_output( - tx_ref, - output.output_index(), - from_account, - output.recipient(), - output.value(), - output.memo(), - ) + let mut stmt_insert_sent_output = conn.prepare_cached( + "INSERT INTO sent_notes ( + tx, output_pool, output_index, from_account, + to_address, to_account, value, memo) + VALUES ( + :tx, :output_pool, :output_index, :from_account, + :to_address, :to_account, :value, :memo)", + )?; + + let (to_address, to_account, pool_type) = recipient_params(params, output.recipient()); + let sql_args = named_params![ + ":tx": &tx_ref, + ":output_pool": &pool_code(pool_type), + ":output_index": &i64::try_from(output.output_index()).unwrap(), + ":from_account": &u32::from(from_account), + ":to_address": &to_address, + ":to_account": &to_account, + ":value": &i64::from(output.value()), + ":memo": output.memo().filter(|m| *m != &MemoBytes::empty()).map(|m| m.as_slice()), + ]; + + stmt_insert_sent_output.execute(sql_args)?; + + Ok(()) } /// Records information about a transaction output that your wallet created. @@ -915,7 +955,8 @@ pub(crate) fn insert_sent_output<'a, P: consensus::Parameters>( /// This is a crate-internal convenience method. #[allow(clippy::too_many_arguments)] pub(crate) fn put_sent_output<'a, P: consensus::Parameters>( - stmts: &mut DataConnStmtCache<'a, P>, + conn: &Connection, + params: &P, from_account: AccountId, tx_ref: i64, output_index: usize, @@ -923,16 +964,34 @@ pub(crate) fn put_sent_output<'a, P: consensus::Parameters>( value: Amount, memo: Option<&MemoBytes>, ) -> Result<(), SqliteClientError> { - if !stmts.stmt_update_sent_output(from_account, recipient, value, memo, tx_ref, output_index)? { - stmts.stmt_insert_sent_output( - tx_ref, - output_index, - from_account, - recipient, - value, - memo, - )?; - } + let mut stmt_upsert_sent_output = conn.prepare_cached( + "INSERT INTO sent_notes ( + tx, output_pool, output_index, from_account, + to_address, to_account, value, memo) + VALUES ( + :tx, :output_pool, :output_index, :from_account, + :to_address, :to_account, :value, :memo) + ON CONFLICT (tx, output_pool, output_index) DO UPDATE + SET from_account = :from_account, + to_address = :to_address, + to_account = :to_account, + value = :value, + memo = IFNULL(:memo, memo)", + )?; + + let (to_address, to_account, pool_type) = recipient_params(params, recipient); + let sql_args = named_params![ + ":tx": &tx_ref, + ":output_pool": &pool_code(pool_type), + ":output_index": &i64::try_from(output_index).unwrap(), + ":from_account": &u32::from(from_account), + ":to_address": &to_address, + ":to_account": &to_account, + ":value": &i64::from(value), + ":memo": &memo.filter(|m| *m != &MemoBytes::empty()).map(|m| m.as_slice()), + ]; + + stmt_upsert_sent_output.execute(sql_args)?; Ok(()) } From 9f6831ea75d76a7c3d9f4e74a7d428c1f0882d1a Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 7 Jun 2023 16:00:27 -0600 Subject: [PATCH 0005/1122] zcash_client_sqlite: Use cached statements for `insert_address` --- zcash_client_sqlite/src/lib.rs | 8 +++- zcash_client_sqlite/src/prepared.rs | 73 +---------------------------- zcash_client_sqlite/src/wallet.rs | 51 ++++++++++++++++---- 3 files changed, 51 insertions(+), 81 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index c2361d29fc..bfe9135a84 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -502,7 +502,13 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { .find_address(search_from) .ok_or(SqliteClientError::DiversifierIndexOutOfRange)?; - self.stmt_insert_address(account, diversifier_index, &addr)?; + wallet::insert_address( + &self.wallet_db.conn, + &self.wallet_db.params, + account, + diversifier_index, + &addr, + )?; Ok(Some(addr)) } diff --git a/zcash_client_sqlite/src/prepared.rs b/zcash_client_sqlite/src/prepared.rs index 1f6779931e..4c86e7ca19 100644 --- a/zcash_client_sqlite/src/prepared.rs +++ b/zcash_client_sqlite/src/prepared.rs @@ -12,10 +12,10 @@ use zcash_primitives::{ consensus::{self, BlockHeight}, merkle_tree::write_incremental_witness, sapling, - zip32::{AccountId, DiversifierIndex}, + zip32::AccountId, }; -use zcash_client_backend::{address::UnifiedAddress, encoding::AddressCodec}; +use zcash_client_backend::encoding::AddressCodec; use crate::{error::SqliteClientError, NoteId, WalletDb}; @@ -26,53 +26,6 @@ use { zcash_primitives::transaction::components::transparent::OutPoint, }; -pub(crate) struct InsertAddress<'a> { - stmt: Statement<'a>, -} - -impl<'a> InsertAddress<'a> { - pub(crate) fn new(conn: &'a rusqlite::Connection) -> Result { - Ok(InsertAddress { - stmt: conn.prepare( - "INSERT INTO addresses ( - account, - diversifier_index_be, - address, - cached_transparent_receiver_address - ) - VALUES ( - :account, - :diversifier_index_be, - :address, - :cached_transparent_receiver_address - )", - )?, - }) - } - - /// Adds the given address and diversifier index to the addresses table. - /// - /// Returns the database row for the newly-inserted address. - pub(crate) fn execute( - &mut self, - params: &P, - account: AccountId, - mut diversifier_index: DiversifierIndex, - address: &UnifiedAddress, - ) -> Result<(), rusqlite::Error> { - // the diversifier index is stored in big-endian order to allow sorting - diversifier_index.0.reverse(); - self.stmt.execute(named_params![ - ":account": &u32::from(account), - ":diversifier_index_be": &&diversifier_index.0[..], - ":address": &address.encode(params), - ":cached_transparent_receiver_address": &address.transparent().map(|r| r.encode(params)), - ])?; - - Ok(()) - } -} - /// The primary type used to implement [`WalletWrite`] for the SQLite database. /// /// A data structure that stores the SQLite prepared statements that are @@ -98,8 +51,6 @@ pub struct DataConnStmtCache<'a, P> { stmt_insert_witness: Statement<'a>, stmt_prune_witnesses: Statement<'a>, stmt_update_expired: Statement<'a>, - - stmt_insert_address: InsertAddress<'a>, } impl<'a, P> DataConnStmtCache<'a, P> { @@ -178,7 +129,6 @@ impl<'a, P> DataConnStmtCache<'a, P> { WHERE id_tx = sapling_received_notes.spent AND block IS NULL AND expiry_height < ? )", )?, - stmt_insert_address: InsertAddress::new(&wallet_db.conn)? } ) } @@ -323,25 +273,6 @@ impl<'a, P: consensus::Parameters> DataConnStmtCache<'a, P> { .optional() .map_err(SqliteClientError::from) } - - /// Adds the given address and diversifier index to the addresses table. - /// - /// Returns the database row for the newly-inserted address. - pub(crate) fn stmt_insert_address( - &mut self, - account: AccountId, - diversifier_index: DiversifierIndex, - address: &UnifiedAddress, - ) -> Result<(), SqliteClientError> { - self.stmt_insert_address.execute( - &self.wallet_db.params, - account, - diversifier_index, - address, - )?; - - Ok(()) - } } impl<'a, P> DataConnStmtCache<'a, P> { diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 8404145d26..9fac4a5fc7 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -64,7 +64,7 @@ //! wallet. //! - `memo` the shielded memo associated with the output, if any. -use rusqlite::{named_params, params, OptionalExtension, ToSql}; +use rusqlite::{named_params, params, Connection, OptionalExtension, ToSql}; use std::collections::HashMap; use std::convert::TryFrom; @@ -84,22 +84,18 @@ use zcash_primitives::{ use zcash_client_backend::{ address::{RecipientAddress, UnifiedAddress}, data_api::{PoolType, Recipient, SentTransactionOutput}, + encoding::AddressCodec, keys::UnifiedFullViewingKey, wallet::WalletTx, }; -use crate::{ - error::SqliteClientError, prepared::InsertAddress, DataConnStmtCache, WalletDb, PRUNING_HEIGHT, -}; +use crate::{error::SqliteClientError, DataConnStmtCache, WalletDb, PRUNING_HEIGHT}; #[cfg(feature = "transparent-inputs")] use { crate::UtxoId, - rusqlite::Connection, std::collections::BTreeSet, - zcash_client_backend::{ - address::AddressMetadata, encoding::AddressCodec, wallet::WalletTransparentOutput, - }, + zcash_client_backend::{address::AddressMetadata, wallet::WalletTransparentOutput}, zcash_primitives::{ legacy::{keys::IncomingViewingKey, Script, TransparentAddress}, transaction::components::{OutPoint, TxOut}, @@ -157,7 +153,7 @@ pub(crate) fn add_account_internal( .transpose() } +/// Adds the given address and diversifier index to the addresses table. +/// +/// Returns the database row for the newly-inserted address. +pub(crate) fn insert_address( + conn: &Connection, + params: &P, + account: AccountId, + mut diversifier_index: DiversifierIndex, + address: &UnifiedAddress, +) -> Result<(), rusqlite::Error> { + let mut stmt = conn.prepare_cached( + "INSERT INTO addresses ( + account, + diversifier_index_be, + address, + cached_transparent_receiver_address + ) + VALUES ( + :account, + :diversifier_index_be, + :address, + :cached_transparent_receiver_address + )", + )?; + + // the diversifier index is stored in big-endian order to allow sorting + diversifier_index.0.reverse(); + stmt.execute(named_params![ + ":account": &u32::from(account), + ":diversifier_index_be": &&diversifier_index.0[..], + ":address": &address.encode(params), + ":cached_transparent_receiver_address": &address.transparent().map(|r| r.encode(params)), + ])?; + + Ok(()) +} + #[cfg(feature = "transparent-inputs")] pub(crate) fn get_transparent_receivers( params: &P, From 21f82e539fcf9b18904719f35a90091936998cda Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 8 Jun 2023 08:53:26 -0600 Subject: [PATCH 0006/1122] zcash_client_sqlite: Use cached prepared statements for transparent UTXO operations. --- zcash_client_sqlite/src/lib.rs | 10 +- zcash_client_sqlite/src/prepared.rs | 228 +--------------------------- zcash_client_sqlite/src/wallet.rs | 143 ++++++++++++----- 3 files changed, 110 insertions(+), 271 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index bfe9135a84..caef02e32e 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -628,7 +628,7 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { // If any of the utxos spent in the transaction are ours, mark them as spent. #[cfg(feature = "transparent-inputs")] for txin in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vin.iter()) { - wallet::mark_transparent_utxo_spent(up, tx_ref, &txin.prevout)?; + wallet::mark_transparent_utxo_spent(&up.wallet_db.conn, tx_ref, &txin.prevout)?; } // If we have some transparent outputs: @@ -692,7 +692,7 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { #[cfg(feature = "transparent-inputs")] for utxo_outpoint in &sent_tx.utxos_spent { - wallet::mark_transparent_utxo_spent(up, tx_ref, utxo_outpoint)?; + wallet::mark_transparent_utxo_spent(&up.wallet_db.conn, tx_ref, utxo_outpoint)?; } for output in &sent_tx.outputs { @@ -735,7 +735,11 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { _output: &WalletTransparentOutput, ) -> Result { #[cfg(feature = "transparent-inputs")] - return wallet::put_received_transparent_utxo(self, _output); + return wallet::put_received_transparent_utxo( + &self.wallet_db.conn, + &self.wallet_db.params, + _output, + ); #[cfg(not(feature = "transparent-inputs"))] panic!( diff --git a/zcash_client_sqlite/src/prepared.rs b/zcash_client_sqlite/src/prepared.rs index 4c86e7ca19..1304b39dd5 100644 --- a/zcash_client_sqlite/src/prepared.rs +++ b/zcash_client_sqlite/src/prepared.rs @@ -7,25 +7,11 @@ //! - Build the statement in [`DataConnStmtCache::new`]. //! - Add a crate-private helper method to `DataConnStmtCache` for running the statement. -use rusqlite::{named_params, params, Statement, ToSql}; -use zcash_primitives::{ - consensus::{self, BlockHeight}, - merkle_tree::write_incremental_witness, - sapling, - zip32::AccountId, -}; - -use zcash_client_backend::encoding::AddressCodec; +use rusqlite::{params, Statement}; +use zcash_primitives::{consensus::BlockHeight, merkle_tree::write_incremental_witness, sapling}; use crate::{error::SqliteClientError, NoteId, WalletDb}; -#[cfg(feature = "transparent-inputs")] -use { - crate::UtxoId, rusqlite::OptionalExtension, - zcash_client_backend::wallet::WalletTransparentOutput, - zcash_primitives::transaction::components::transparent::OutPoint, -}; - /// The primary type used to implement [`WalletWrite`] for the SQLite database. /// /// A data structure that stores the SQLite prepared statements that are @@ -36,18 +22,6 @@ use { pub struct DataConnStmtCache<'a, P> { pub(crate) wallet_db: &'a WalletDb

, - #[cfg(feature = "transparent-inputs")] - stmt_mark_transparent_utxo_spent: Statement<'a>, - - #[cfg(feature = "transparent-inputs")] - stmt_insert_received_transparent_utxo: Statement<'a>, - #[cfg(feature = "transparent-inputs")] - stmt_update_received_transparent_utxo: Statement<'a>, - #[cfg(feature = "transparent-inputs")] - stmt_insert_legacy_transparent_utxo: Statement<'a>, - #[cfg(feature = "transparent-inputs")] - stmt_update_legacy_transparent_utxo: Statement<'a>, - stmt_insert_witness: Statement<'a>, stmt_prune_witnesses: Statement<'a>, stmt_update_expired: Statement<'a>, @@ -59,63 +33,6 @@ impl<'a, P> DataConnStmtCache<'a, P> { DataConnStmtCache { wallet_db, #[cfg(feature = "transparent-inputs")] - stmt_mark_transparent_utxo_spent: wallet_db.conn.prepare( - "UPDATE utxos SET spent_in_tx = :spent_in_tx - WHERE prevout_txid = :prevout_txid - AND prevout_idx = :prevout_idx" - )?, - #[cfg(feature = "transparent-inputs")] - stmt_insert_received_transparent_utxo: wallet_db.conn.prepare( - "INSERT INTO utxos ( - received_by_account, address, - prevout_txid, prevout_idx, script, - value_zat, height) - SELECT - addresses.account, :address, - :prevout_txid, :prevout_idx, :script, - :value_zat, :height - FROM addresses - WHERE addresses.cached_transparent_receiver_address = :address - RETURNING id_utxo" - )?, - #[cfg(feature = "transparent-inputs")] - stmt_update_received_transparent_utxo: wallet_db.conn.prepare( - "UPDATE utxos - SET received_by_account = addresses.account, - height = :height, - address = :address, - script = :script, - value_zat = :value_zat - FROM addresses - WHERE prevout_txid = :prevout_txid - AND prevout_idx = :prevout_idx - AND addresses.cached_transparent_receiver_address = :address - RETURNING id_utxo" - )?, - #[cfg(feature = "transparent-inputs")] - stmt_insert_legacy_transparent_utxo: wallet_db.conn.prepare( - "INSERT INTO utxos ( - received_by_account, address, - prevout_txid, prevout_idx, script, - value_zat, height) - VALUES - (:received_by_account, :address, - :prevout_txid, :prevout_idx, :script, - :value_zat, :height) - RETURNING id_utxo" - )?, - #[cfg(feature = "transparent-inputs")] - stmt_update_legacy_transparent_utxo: wallet_db.conn.prepare( - "UPDATE utxos - SET received_by_account = :received_by_account, - height = :height, - address = :address, - script = :script, - value_zat = :value_zat - WHERE prevout_txid = :prevout_txid - AND prevout_idx = :prevout_idx - RETURNING id_utxo" - )?, stmt_insert_witness: wallet_db.conn.prepare( "INSERT INTO sapling_witnesses (note, block, witness) VALUES (?, ?, ?)", @@ -132,147 +49,6 @@ impl<'a, P> DataConnStmtCache<'a, P> { } ) } - - /// Marks the given UTXO as having been spent. - /// - /// Returns `false` if `outpoint` does not correspond to any tracked UTXO. - #[cfg(feature = "transparent-inputs")] - pub(crate) fn stmt_mark_transparent_utxo_spent( - &mut self, - tx_ref: i64, - outpoint: &OutPoint, - ) -> Result { - let sql_args: &[(&str, &dyn ToSql)] = &[ - (":spent_in_tx", &tx_ref), - (":prevout_txid", &outpoint.hash().to_vec()), - (":prevout_idx", &outpoint.n()), - ]; - - match self.stmt_mark_transparent_utxo_spent.execute(sql_args)? { - 0 => Ok(false), - 1 => Ok(true), - _ => unreachable!("tx_outpoint constraint is marked as UNIQUE"), - } - } -} - -impl<'a, P: consensus::Parameters> DataConnStmtCache<'a, P> { - /// Adds the given received UTXO to the datastore. - /// - /// Returns the database identifier for the newly-inserted UTXO if the address to which the - /// UTXO was sent corresponds to a cached transparent receiver in the addresses table, or - /// Ok(None) if the address is unknown. Returns an error if the UTXO exists. - #[cfg(feature = "transparent-inputs")] - pub(crate) fn stmt_insert_received_transparent_utxo( - &mut self, - output: &WalletTransparentOutput, - ) -> Result, SqliteClientError> { - self.stmt_insert_received_transparent_utxo - .query_row( - named_params![ - ":address": &output.recipient_address().encode(&self.wallet_db.params), - ":prevout_txid": &output.outpoint().hash().to_vec(), - ":prevout_idx": &output.outpoint().n(), - ":script": &output.txout().script_pubkey.0, - ":value_zat": &i64::from(output.txout().value), - ":height": &u32::from(output.height()), - ], - |row| { - let id = row.get(0)?; - Ok(UtxoId(id)) - }, - ) - .optional() - .map_err(SqliteClientError::from) - } - - /// Adds the given received UTXO to the datastore. - /// - /// Returns the database identifier for the updated UTXO if the address to which the UTXO was - /// sent corresponds to a cached transparent receiver in the addresses table, or Ok(None) if - /// the address is unknown. - #[cfg(feature = "transparent-inputs")] - pub(crate) fn stmt_update_received_transparent_utxo( - &mut self, - output: &WalletTransparentOutput, - ) -> Result, SqliteClientError> { - self.stmt_update_received_transparent_utxo - .query_row( - named_params![ - ":prevout_txid": &output.outpoint().hash().to_vec(), - ":prevout_idx": &output.outpoint().n(), - ":address": &output.recipient_address().encode(&self.wallet_db.params), - ":script": &output.txout().script_pubkey.0, - ":value_zat": &i64::from(output.txout().value), - ":height": &u32::from(output.height()), - ], - |row| { - let id = row.get(0)?; - Ok(UtxoId(id)) - }, - ) - .optional() - .map_err(SqliteClientError::from) - } - - /// Adds the given legacy UTXO to the datastore. - /// - /// Returns the database row for the newly-inserted UTXO, or an error if the UTXO - /// exists. - #[cfg(feature = "transparent-inputs")] - pub(crate) fn stmt_insert_legacy_transparent_utxo( - &mut self, - output: &WalletTransparentOutput, - received_by_account: AccountId, - ) -> Result { - self.stmt_insert_legacy_transparent_utxo - .query_row( - named_params![ - ":received_by_account": &u32::from(received_by_account), - ":address": &output.recipient_address().encode(&self.wallet_db.params), - ":prevout_txid": &output.outpoint().hash().to_vec(), - ":prevout_idx": &output.outpoint().n(), - ":script": &output.txout().script_pubkey.0, - ":value_zat": &i64::from(output.txout().value), - ":height": &u32::from(output.height()), - ], - |row| { - let id = row.get(0)?; - Ok(UtxoId(id)) - }, - ) - .map_err(SqliteClientError::from) - } - - /// Adds the given legacy UTXO to the datastore. - /// - /// Returns the database row for the newly-inserted UTXO, or an error if the UTXO - /// exists. - #[cfg(feature = "transparent-inputs")] - pub(crate) fn stmt_update_legacy_transparent_utxo( - &mut self, - output: &WalletTransparentOutput, - received_by_account: AccountId, - ) -> Result, SqliteClientError> { - self.stmt_update_legacy_transparent_utxo - .query_row( - named_params![ - ":received_by_account": &u32::from(received_by_account), - ":prevout_txid": &output.outpoint().hash().to_vec(), - ":prevout_idx": &output.outpoint().n(), - ":address": &output.recipient_address().encode(&self.wallet_db.params), - ":script": &output.txout().script_pubkey.0, - ":value_zat": &i64::from(output.txout().value), - ":height": &u32::from(output.height()), - ], - |row| { - let id = row.get(0)?; - Ok(UtxoId(id)) - }, - ) - .optional() - .map_err(SqliteClientError::from) - } } impl<'a, P> DataConnStmtCache<'a, P> { diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 9fac4a5fc7..884e672fb1 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -855,56 +855,106 @@ pub(crate) fn put_tx_data<'a, P>( /// Marks the given UTXO as having been spent. #[cfg(feature = "transparent-inputs")] -pub(crate) fn mark_transparent_utxo_spent<'a, P>( - stmts: &mut DataConnStmtCache<'a, P>, +pub(crate) fn mark_transparent_utxo_spent( + conn: &Connection, tx_ref: i64, outpoint: &OutPoint, ) -> Result<(), SqliteClientError> { - stmts.stmt_mark_transparent_utxo_spent(tx_ref, outpoint)?; + let mut stmt_mark_transparent_utxo_spent = conn.prepare_cached( + "UPDATE utxos SET spent_in_tx = :spent_in_tx + WHERE prevout_txid = :prevout_txid + AND prevout_idx = :prevout_idx", + )?; + let sql_args = named_params![ + ":spent_in_tx": &tx_ref, + ":prevout_txid": &outpoint.hash().to_vec(), + ":prevout_idx": &outpoint.n(), + ]; + + stmt_mark_transparent_utxo_spent.execute(sql_args)?; Ok(()) } /// Adds the given received UTXO to the datastore. #[cfg(feature = "transparent-inputs")] -pub(crate) fn put_received_transparent_utxo<'a, P: consensus::Parameters>( - stmts: &mut DataConnStmtCache<'a, P>, +pub(crate) fn put_received_transparent_utxo( + conn: &Connection, + params: &P, output: &WalletTransparentOutput, ) -> Result { - stmts - .stmt_update_received_transparent_utxo(output) - .transpose() - .or_else(|| { - stmts - .stmt_insert_received_transparent_utxo(output) - .transpose() - }) - .unwrap_or_else(|| { - // This could occur if the UTXO is received at the legacy transparent - // address, in which case the join to the `addresses` table will fail. - // In this case, we should look up the legacy address for account 0 and - // check whether it matches the address for the received UTXO, and if - // so then insert/update it directly. - let account = AccountId::from(0u32); - get_legacy_transparent_address(&stmts.wallet_db.params, &stmts.wallet_db.conn, account) - .and_then(|legacy_taddr| { - if legacy_taddr - .iter() - .any(|(taddr, _)| taddr == output.recipient_address()) - { - stmts - .stmt_update_legacy_transparent_utxo(output, account) - .transpose() - .unwrap_or_else(|| { - stmts.stmt_insert_legacy_transparent_utxo(output, account) - }) - } else { - Err(SqliteClientError::AddressNotRecognized( - *output.recipient_address(), - )) - } - }) - }) + let address_str = output.recipient_address().encode(params); + let account_id = conn + .query_row( + "SELECT account FROM addresses WHERE cached_transparent_receiver_address = :address", + named_params![":address": &address_str], + |row| row.get::<_, u32>(0).map(AccountId::from), + ) + .optional()?; + + let utxoid = if let Some(account) = account_id { + put_legacy_transparent_utxo(conn, params, output, account)? + } else { + // If the UTXO is received at the legacy transparent address, there may be no entry in the + // addresses table that can be used to tie the address to a particular account. In this + // case, we should look up the legacy address for account 0 and check whether it matches + // the address for the received UTXO, and if so then insert/update it directly. + let account = AccountId::from(0u32); + get_legacy_transparent_address(params, conn, account).and_then(|legacy_taddr| { + if legacy_taddr + .iter() + .any(|(taddr, _)| taddr == output.recipient_address()) + { + put_legacy_transparent_utxo(conn, params, output, account) + .map_err(SqliteClientError::from) + } else { + Err(SqliteClientError::AddressNotRecognized( + *output.recipient_address(), + )) + } + })? + }; + + Ok(utxoid) +} + +#[cfg(feature = "transparent-inputs")] +pub(crate) fn put_legacy_transparent_utxo( + conn: &Connection, + params: &P, + output: &WalletTransparentOutput, + received_by_account: AccountId, +) -> Result { + #[cfg(feature = "transparent-inputs")] + let mut stmt_upsert_legacy_transparent_utxo = conn.prepare_cached( + "INSERT INTO utxos ( + prevout_txid, prevout_idx, + received_by_account, address, script, + value_zat, height) + VALUES + (:prevout_txid, :prevout_idx, + :received_by_account, :address, :script, + :value_zat, :height) + ON CONFLICT (prevout_txid, prevout_idx) DO UPDATE + SET received_by_account = :received_by_account, + height = :height, + address = :address, + script = :script, + value_zat = :value_zat + RETURNING id_utxo", + )?; + + let sql_args = named_params![ + ":prevout_txid": &output.outpoint().hash().to_vec(), + ":prevout_idx": &output.outpoint().n(), + ":received_by_account": &u32::from(received_by_account), + ":address": &output.recipient_address().encode(params), + ":script": &output.txout().script_pubkey.0, + ":value_zat": &i64::from(output.txout().value), + ":height": &u32::from(output.height()), + ]; + + stmt_upsert_legacy_transparent_utxo.query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId)) } /// Removes old incremental witnesses up to the given block height. @@ -1112,7 +1162,8 @@ mod tests { ) .unwrap(); - let res0 = super::put_received_transparent_utxo(&mut ops, &utxo); + let res0 = + super::put_received_transparent_utxo(&ops.wallet_db.conn, &ops.wallet_db.params, &utxo); assert_matches!(res0, Ok(_)); // Change the mined height of the UTXO and upsert; we should get back @@ -1126,7 +1177,11 @@ mod tests { BlockHeight::from_u32(34567), ) .unwrap(); - let res1 = super::put_received_transparent_utxo(&mut ops, &utxo2); + let res1 = super::put_received_transparent_utxo( + &ops.wallet_db.conn, + &ops.wallet_db.params, + &utxo2, + ); assert_matches!(res1, Ok(id) if id == res0.unwrap()); assert_matches!( @@ -1167,7 +1222,11 @@ mod tests { ) .unwrap(); - let res2 = super::put_received_transparent_utxo(&mut ops, &utxo2); + let res2 = super::put_received_transparent_utxo( + &ops.wallet_db.conn, + &ops.wallet_db.params, + &utxo2, + ); assert_matches!(res2, Err(_)); } } From bf7f05282fb8814806662f8ff1855964065199b8 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 8 Jun 2023 12:53:39 -0600 Subject: [PATCH 0007/1122] zcash_client_sqlite: Fix missing incrementalmerkletree feature dependency --- zcash_client_sqlite/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index c58dba2a7d..80a2000269 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -47,6 +47,7 @@ uuid = "1.1" [dev-dependencies] assert_matches = "1.5" +incrementalmerkletree = { version = "0.4", features = ["legacy-api", "test-dependencies"] } proptest = "1.0.0" rand_core = "0.6" regex = "1.4" @@ -59,6 +60,7 @@ zcash_address = { version = "0.3", path = "../components/zcash_address", feature [features] mainnet = [] test-dependencies = [ + "incrementalmerkletree/test-dependencies", "zcash_primitives/test-dependencies", "zcash_client_backend/test-dependencies", ] From 2674209818378584bb1a98276e4e27a9b11882b6 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 9 Jun 2023 11:02:00 -0600 Subject: [PATCH 0008/1122] zcash_client_sqlite: Remove the remainder of DataConnStmtCache --- zcash_client_sqlite/src/chain.rs | 97 +-- zcash_client_sqlite/src/lib.rs | 556 +++++++----------- zcash_client_sqlite/src/prepared.rs | 95 --- zcash_client_sqlite/src/wallet.rs | 341 ++++++----- zcash_client_sqlite/src/wallet/init.rs | 123 ++-- .../init/migrations/add_utxo_account.rs | 2 +- zcash_client_sqlite/src/wallet/sapling.rs | 233 ++++---- 7 files changed, 631 insertions(+), 816 deletions(-) delete mode 100644 zcash_client_sqlite/src/prepared.rs diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index fc9e8d09f2..81a0e028a6 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -299,7 +299,7 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&db_data); + let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); // Empty chain should return None assert_matches!(db_data.get_max_height_hash(), Ok(None)); @@ -328,8 +328,7 @@ mod tests { assert_matches!(validate_chain_result, Ok(())); // Scan the cache - let mut db_write = db_data.get_update_ops().unwrap(); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -348,7 +347,7 @@ mod tests { validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -365,7 +364,7 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&db_data); + let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); // Create some fake CompactBlocks let (cb, _) = fake_compact_block( @@ -386,8 +385,7 @@ mod tests { insert_into_cache(&db_cache, &cb2); // Scan the cache - let mut db_write = db_data.get_update_ops().unwrap(); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -427,7 +425,7 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&db_data); + let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); // Create some fake CompactBlocks let (cb, _) = fake_compact_block( @@ -448,8 +446,7 @@ mod tests { insert_into_cache(&db_cache, &cb2); // Scan the cache - let mut db_write = db_data.get_update_ops().unwrap(); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -489,11 +486,11 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&db_data); + let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); // Account balance should be zero assert_eq!( - get_balance(&db_data, AccountId::from(0)).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), Amount::zero() ); @@ -519,36 +516,46 @@ mod tests { insert_into_cache(&db_cache, &cb2); // Scan the cache - let mut db_write = db_data.get_update_ops().unwrap(); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Account balance should reflect both received notes assert_eq!( - get_balance(&db_data, AccountId::from(0)).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), (value + value2).unwrap() ); // "Rewind" to height of last scanned block - truncate_to_height(&db_data, sapling_activation_height() + 1).unwrap(); + db_data + .transactionally(|wdb| { + truncate_to_height(&wdb.conn.0, &wdb.params, sapling_activation_height() + 1) + }) + .unwrap(); // Account balance should be unaltered assert_eq!( - get_balance(&db_data, AccountId::from(0)).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), (value + value2).unwrap() ); // Rewind so that one block is dropped - truncate_to_height(&db_data, sapling_activation_height()).unwrap(); + db_data + .transactionally(|wdb| { + truncate_to_height(&wdb.conn.0, &wdb.params, sapling_activation_height()) + }) + .unwrap(); // Account balance should only contain the first received note - assert_eq!(get_balance(&db_data, AccountId::from(0)).unwrap(), value); + assert_eq!( + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + value + ); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Account balance should again reflect both received notes assert_eq!( - get_balance(&db_data, AccountId::from(0)).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), (value + value2).unwrap() ); } @@ -564,7 +571,7 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&db_data); + let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); // Create a block with height SAPLING_ACTIVATION_HEIGHT let value = Amount::from_u64(50000).unwrap(); @@ -576,9 +583,11 @@ mod tests { value, ); insert_into_cache(&db_cache, &cb1); - let mut db_write = db_data.get_update_ops().unwrap(); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); - assert_eq!(get_balance(&db_data, AccountId::from(0)).unwrap(), value); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + assert_eq!( + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + value + ); // We cannot scan a block of height SAPLING_ACTIVATION_HEIGHT + 2 next let (cb2, _) = fake_compact_block( @@ -596,7 +605,7 @@ mod tests { value, ); insert_into_cache(&db_cache, &cb3); - match scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None) { + match scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None) { Err(Error::Chain(e)) => { assert_matches!( e.cause(), @@ -609,9 +618,9 @@ mod tests { // If we add a block of height SAPLING_ACTIVATION_HEIGHT + 1, we can now scan both insert_into_cache(&db_cache, &cb2); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); assert_eq!( - get_balance(&db_data, AccountId::from(0)).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), Amount::from_u64(150_000).unwrap() ); } @@ -627,11 +636,11 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&db_data); + let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); // Account balance should be zero assert_eq!( - get_balance(&db_data, AccountId::from(0)).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), Amount::zero() ); @@ -647,11 +656,13 @@ mod tests { insert_into_cache(&db_cache, &cb); // Scan the cache - let mut db_write = db_data.get_update_ops().unwrap(); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Account balance should reflect the received note - assert_eq!(get_balance(&db_data, AccountId::from(0)).unwrap(), value); + assert_eq!( + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + value + ); // Create a second fake CompactBlock sending more value to the address let value2 = Amount::from_u64(7).unwrap(); @@ -665,11 +676,11 @@ mod tests { insert_into_cache(&db_cache, &cb2); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Account balance should reflect both received notes assert_eq!( - get_balance(&db_data, AccountId::from(0)).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), (value + value2).unwrap() ); } @@ -685,11 +696,11 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&db_data); + let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); // Account balance should be zero assert_eq!( - get_balance(&db_data, AccountId::from(0)).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), Amount::zero() ); @@ -705,11 +716,13 @@ mod tests { insert_into_cache(&db_cache, &cb); // Scan the cache - let mut db_write = db_data.get_update_ops().unwrap(); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Account balance should reflect the received note - assert_eq!(get_balance(&db_data, AccountId::from(0)).unwrap(), value); + assert_eq!( + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + value + ); // Create a second fake CompactBlock spending value from the address let extsk2 = ExtendedSpendingKey::master(&[0]); @@ -728,11 +741,11 @@ mod tests { ); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Account balance should equal the change assert_eq!( - get_balance(&db_data, AccountId::from(0)).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), (value - value2).unwrap() ); } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index caef02e32e..e3b13daf34 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -32,11 +32,9 @@ // Catch documentation errors caused by code changes. #![deny(rustdoc::broken_intra_doc_links)] -use rusqlite::Connection; +use rusqlite::{self, Connection}; use secrecy::{ExposeSecret, SecretVec}; -use std::collections::HashMap; -use std::fmt; -use std::path::Path; +use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, path::Path}; use zcash_primitives::{ block::BlockHash, @@ -72,9 +70,6 @@ use { std::{fs, io}, }; -mod prepared; -pub use prepared::DataConnStmtCache; - pub mod chain; pub mod error; pub mod wallet; @@ -107,12 +102,21 @@ impl fmt::Display for NoteId { pub struct UtxoId(pub i64); /// A wrapper for the SQLite connection to the wallet database. -pub struct WalletDb

{ - conn: Connection, +pub struct WalletDb { + conn: C, params: P, } -impl WalletDb

{ +/// A wrapper for a SQLite transaction affecting the wallet database. +pub struct WalletTransaction<'conn>(pub(crate) rusqlite::Transaction<'conn>); + +impl Borrow for WalletTransaction<'_> { + fn borrow(&self) -> &rusqlite::Connection { + &self.0 + } +} + +impl WalletDb { /// Construct a connection to the wallet database stored at the specified path. pub fn for_path>(path: F, params: P) -> Result { Connection::open(path).and_then(move |conn| { @@ -121,53 +125,60 @@ impl WalletDb

{ }) } - /// Given a wallet database connection, obtain a handle for the write operations - /// for that database. This operation may eagerly initialize and cache sqlite - /// prepared statements that are used in write operations. - pub fn get_update_ops(&self) -> Result, SqliteClientError> { - DataConnStmtCache::new(self) + pub fn transactionally(&mut self, f: F) -> Result + where + F: FnOnce(&WalletDb, P>) -> Result, + { + let wdb = WalletDb { + conn: WalletTransaction(self.conn.transaction()?), + params: self.params.clone(), + }; + let result = f(&wdb)?; + wdb.conn.0.commit()?; + Ok(result) } } -impl WalletRead for WalletDb

{ +impl, P: consensus::Parameters> WalletRead for WalletDb { type Error = SqliteClientError; type NoteRef = NoteId; type TxRef = i64; fn block_height_extrema(&self) -> Result, Self::Error> { - wallet::block_height_extrema(self).map_err(SqliteClientError::from) + wallet::block_height_extrema(self.conn.borrow()).map_err(SqliteClientError::from) } fn get_min_unspent_height(&self) -> Result, Self::Error> { - wallet::get_min_unspent_height(self).map_err(SqliteClientError::from) + wallet::get_min_unspent_height(self.conn.borrow()).map_err(SqliteClientError::from) } fn get_block_hash(&self, block_height: BlockHeight) -> Result, Self::Error> { - wallet::get_block_hash(self, block_height).map_err(SqliteClientError::from) + wallet::get_block_hash(self.conn.borrow(), block_height).map_err(SqliteClientError::from) } fn get_tx_height(&self, txid: TxId) -> Result, Self::Error> { - wallet::get_tx_height(self, txid).map_err(SqliteClientError::from) + wallet::get_tx_height(self.conn.borrow(), txid).map_err(SqliteClientError::from) } fn get_unified_full_viewing_keys( &self, ) -> Result, Self::Error> { - wallet::get_unified_full_viewing_keys(self) + wallet::get_unified_full_viewing_keys(self.conn.borrow(), &self.params) } fn get_account_for_ufvk( &self, ufvk: &UnifiedFullViewingKey, ) -> Result, Self::Error> { - wallet::get_account_for_ufvk(self, ufvk) + wallet::get_account_for_ufvk(self.conn.borrow(), &self.params, ufvk) } fn get_current_address( &self, account: AccountId, ) -> Result, Self::Error> { - wallet::get_current_address(self, account).map(|res| res.map(|(addr, _)| addr)) + wallet::get_current_address(self.conn.borrow(), &self.params, account) + .map(|res| res.map(|(addr, _)| addr)) } fn is_valid_account_extfvk( @@ -175,7 +186,7 @@ impl WalletRead for WalletDb

{ account: AccountId, extfvk: &ExtendedFullViewingKey, ) -> Result { - wallet::is_valid_account_extfvk(self, account, extfvk) + wallet::is_valid_account_extfvk(self.conn.borrow(), &self.params, account, extfvk) } fn get_balance_at( @@ -183,17 +194,19 @@ impl WalletRead for WalletDb

{ account: AccountId, anchor_height: BlockHeight, ) -> Result { - wallet::get_balance_at(self, account, anchor_height) + wallet::get_balance_at(self.conn.borrow(), account, anchor_height) } fn get_transaction(&self, id_tx: i64) -> Result { - wallet::get_transaction(self, id_tx) + wallet::get_transaction(self.conn.borrow(), &self.params, id_tx) } fn get_memo(&self, id_note: Self::NoteRef) -> Result, Self::Error> { match id_note { - NoteId::SentNoteId(id_note) => wallet::get_sent_memo(self, id_note), - NoteId::ReceivedNoteId(id_note) => wallet::get_received_memo(self, id_note), + NoteId::SentNoteId(id_note) => wallet::get_sent_memo(self.conn.borrow(), id_note), + NoteId::ReceivedNoteId(id_note) => { + wallet::get_received_memo(self.conn.borrow(), id_note) + } } } @@ -201,7 +214,7 @@ impl WalletRead for WalletDb

{ &self, block_height: BlockHeight, ) -> Result, Self::Error> { - wallet::sapling::get_sapling_commitment_tree(self, block_height) + wallet::sapling::get_sapling_commitment_tree(self.conn.borrow(), block_height) } #[allow(clippy::type_complexity)] @@ -209,7 +222,7 @@ impl WalletRead for WalletDb

{ &self, block_height: BlockHeight, ) -> Result, Self::Error> { - wallet::sapling::get_sapling_witnesses(self, block_height) + wallet::sapling::get_sapling_witnesses(self.conn.borrow(), block_height) } fn get_sapling_nullifiers( @@ -217,8 +230,8 @@ impl WalletRead for WalletDb

{ query: data_api::NullifierQuery, ) -> Result, Self::Error> { match query { - NullifierQuery::Unspent => wallet::sapling::get_sapling_nullifiers(self), - NullifierQuery::All => wallet::sapling::get_all_sapling_nullifiers(&self.conn), + NullifierQuery::Unspent => wallet::sapling::get_sapling_nullifiers(self.conn.borrow()), + NullifierQuery::All => wallet::sapling::get_all_sapling_nullifiers(self.conn.borrow()), } } @@ -228,7 +241,12 @@ impl WalletRead for WalletDb

{ anchor_height: BlockHeight, exclude: &[Self::NoteRef], ) -> Result>, Self::Error> { - wallet::sapling::get_spendable_sapling_notes(&self.conn, account, anchor_height, exclude) + wallet::sapling::get_spendable_sapling_notes( + self.conn.borrow(), + account, + anchor_height, + exclude, + ) } fn select_spendable_sapling_notes( @@ -239,7 +257,7 @@ impl WalletRead for WalletDb

{ exclude: &[Self::NoteRef], ) -> Result>, Self::Error> { wallet::sapling::select_spendable_sapling_notes( - &self.conn, + self.conn.borrow(), account, target_value, anchor_height, @@ -252,7 +270,7 @@ impl WalletRead for WalletDb

{ _account: AccountId, ) -> Result, Self::Error> { #[cfg(feature = "transparent-inputs")] - return wallet::get_transparent_receivers(&self.params, &self.conn, _account); + return wallet::get_transparent_receivers(self.conn.borrow(), &self.params, _account); #[cfg(not(feature = "transparent-inputs"))] panic!( @@ -267,7 +285,13 @@ impl WalletRead for WalletDb

{ _exclude: &[OutPoint], ) -> Result, Self::Error> { #[cfg(feature = "transparent-inputs")] - return wallet::get_unspent_transparent_outputs(self, _address, _max_height, _exclude); + return wallet::get_unspent_transparent_outputs( + self.conn.borrow(), + &self.params, + _address, + _max_height, + _exclude, + ); #[cfg(not(feature = "transparent-inputs"))] panic!( @@ -281,7 +305,12 @@ impl WalletRead for WalletDb

{ _max_height: BlockHeight, ) -> Result, Self::Error> { #[cfg(feature = "transparent-inputs")] - return wallet::get_transparent_balances(self, _account, _max_height); + return wallet::get_transparent_balances( + self.conn.borrow(), + &self.params, + _account, + _max_height, + ); #[cfg(not(feature = "transparent-inputs"))] panic!( @@ -290,177 +319,15 @@ impl WalletRead for WalletDb

{ } } -impl<'a, P: consensus::Parameters> WalletRead for DataConnStmtCache<'a, P> { - type Error = SqliteClientError; - type NoteRef = NoteId; - type TxRef = i64; - - fn block_height_extrema(&self) -> Result, Self::Error> { - self.wallet_db.block_height_extrema() - } - - fn get_min_unspent_height(&self) -> Result, Self::Error> { - self.wallet_db.get_min_unspent_height() - } - - fn get_block_hash(&self, block_height: BlockHeight) -> Result, Self::Error> { - self.wallet_db.get_block_hash(block_height) - } - - fn get_tx_height(&self, txid: TxId) -> Result, Self::Error> { - self.wallet_db.get_tx_height(txid) - } - - fn get_unified_full_viewing_keys( - &self, - ) -> Result, Self::Error> { - self.wallet_db.get_unified_full_viewing_keys() - } - - fn get_account_for_ufvk( - &self, - ufvk: &UnifiedFullViewingKey, - ) -> Result, Self::Error> { - self.wallet_db.get_account_for_ufvk(ufvk) - } - - fn get_current_address( - &self, - account: AccountId, - ) -> Result, Self::Error> { - self.wallet_db.get_current_address(account) - } - - fn is_valid_account_extfvk( - &self, - account: AccountId, - extfvk: &ExtendedFullViewingKey, - ) -> Result { - self.wallet_db.is_valid_account_extfvk(account, extfvk) - } - - fn get_balance_at( - &self, - account: AccountId, - anchor_height: BlockHeight, - ) -> Result { - self.wallet_db.get_balance_at(account, anchor_height) - } - - fn get_transaction(&self, id_tx: i64) -> Result { - self.wallet_db.get_transaction(id_tx) - } - - fn get_memo(&self, id_note: Self::NoteRef) -> Result, Self::Error> { - self.wallet_db.get_memo(id_note) - } - - fn get_commitment_tree( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error> { - self.wallet_db.get_commitment_tree(block_height) - } - - #[allow(clippy::type_complexity)] - fn get_witnesses( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error> { - self.wallet_db.get_witnesses(block_height) - } - - fn get_sapling_nullifiers( - &self, - query: data_api::NullifierQuery, - ) -> Result, Self::Error> { - self.wallet_db.get_sapling_nullifiers(query) - } - - fn get_spendable_sapling_notes( - &self, - account: AccountId, - anchor_height: BlockHeight, - exclude: &[Self::NoteRef], - ) -> Result>, Self::Error> { - self.wallet_db - .get_spendable_sapling_notes(account, anchor_height, exclude) - } - - fn select_spendable_sapling_notes( - &self, - account: AccountId, - target_value: Amount, - anchor_height: BlockHeight, - exclude: &[Self::NoteRef], - ) -> Result>, Self::Error> { - self.wallet_db - .select_spendable_sapling_notes(account, target_value, anchor_height, exclude) - } - - fn get_transparent_receivers( - &self, - account: AccountId, - ) -> Result, Self::Error> { - self.wallet_db.get_transparent_receivers(account) - } - - fn get_unspent_transparent_outputs( - &self, - address: &TransparentAddress, - max_height: BlockHeight, - exclude: &[OutPoint], - ) -> Result, Self::Error> { - self.wallet_db - .get_unspent_transparent_outputs(address, max_height, exclude) - } - - fn get_transparent_balances( - &self, - account: AccountId, - max_height: BlockHeight, - ) -> Result, Self::Error> { - self.wallet_db.get_transparent_balances(account, max_height) - } -} - -impl<'a, P: consensus::Parameters> DataConnStmtCache<'a, P> { - fn transactionally(&mut self, f: F) -> Result - where - F: FnOnce(&mut Self) -> Result, - { - self.wallet_db.conn.execute("BEGIN IMMEDIATE", [])?; - match f(self) { - Ok(result) => { - self.wallet_db.conn.execute("COMMIT", [])?; - Ok(result) - } - Err(error) => { - match self.wallet_db.conn.execute("ROLLBACK", []) { - Ok(_) => Err(error), - Err(e) => - // Panicking here is probably the right thing to do, because it - // means the database is corrupt. - panic!( - "Rollback failed with error {} while attempting to recover from error {}; database is likely corrupt.", - e, - error - ) - } - } - } - } -} - -impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { +impl WalletWrite for WalletDb { type UtxoRef = UtxoId; fn create_account( &mut self, seed: &SecretVec, ) -> Result<(AccountId, UnifiedSpendingKey), Self::Error> { - self.transactionally(|stmts| { - let account = wallet::get_max_account_id(stmts.wallet_db)? + self.transactionally(|wdb| { + let account = wallet::get_max_account_id(&wdb.conn.0)? .map(|a| AccountId::from(u32::from(a) + 1)) .unwrap_or_else(|| AccountId::from(0)); @@ -468,15 +335,11 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { return Err(SqliteClientError::AccountIdOutOfRange); } - let usk = UnifiedSpendingKey::from_seed( - &stmts.wallet_db.params, - seed.expose_secret(), - account, - ) - .map_err(|_| SqliteClientError::KeyDerivationError(account))?; + let usk = UnifiedSpendingKey::from_seed(&wdb.params, seed.expose_secret(), account) + .map_err(|_| SqliteClientError::KeyDerivationError(account))?; let ufvk = usk.to_unified_full_viewing_key(); - wallet::add_account(stmts.wallet_db, account, &ufvk)?; + wallet::add_account(&wdb.conn.0, &wdb.params, account, &ufvk)?; Ok((account, usk)) }) @@ -486,34 +349,37 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { &mut self, account: AccountId, ) -> Result, Self::Error> { - match self.get_unified_full_viewing_keys()?.get(&account) { - Some(ufvk) => { - let search_from = match wallet::get_current_address(self.wallet_db, account)? { - Some((_, mut last_diversifier_index)) => { - last_diversifier_index - .increment() - .map_err(|_| SqliteClientError::DiversifierIndexOutOfRange)?; - last_diversifier_index - } - None => DiversifierIndex::default(), - }; - - let (addr, diversifier_index) = ufvk - .find_address(search_from) - .ok_or(SqliteClientError::DiversifierIndexOutOfRange)?; - - wallet::insert_address( - &self.wallet_db.conn, - &self.wallet_db.params, - account, - diversifier_index, - &addr, - )?; + self.transactionally( + |wdb| match wdb.get_unified_full_viewing_keys()?.get(&account) { + Some(ufvk) => { + let search_from = + match wallet::get_current_address(&wdb.conn.0, &wdb.params, account)? { + Some((_, mut last_diversifier_index)) => { + last_diversifier_index + .increment() + .map_err(|_| SqliteClientError::DiversifierIndexOutOfRange)?; + last_diversifier_index + } + None => DiversifierIndex::default(), + }; - Ok(Some(addr)) - } - None => Ok(None), - } + let (addr, diversifier_index) = ufvk + .find_address(search_from) + .ok_or(SqliteClientError::DiversifierIndexOutOfRange)?; + + wallet::insert_address( + &wdb.conn.0, + &wdb.params, + account, + diversifier_index, + &addr, + )?; + + Ok(Some(addr)) + } + None => Ok(None), + }, + ) } #[tracing::instrument(skip_all, fields(height = u32::from(block.block_height)))] @@ -523,11 +389,10 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { block: &PrunedBlock, updated_witnesses: &[(Self::NoteRef, sapling::IncrementalWitness)], ) -> Result, Self::Error> { - // database updates for each block are transactional - self.transactionally(|up| { + self.transactionally(|wdb| { // Insert the block into the database. wallet::insert_block( - up, + &wdb.conn.0, block.block_height, block.block_hash, block.block_time, @@ -536,20 +401,16 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { let mut new_witnesses = vec![]; for tx in block.transactions { - let tx_row = wallet::put_tx_meta(up, tx, block.block_height)?; + let tx_row = wallet::put_tx_meta(&wdb.conn.0, tx, block.block_height)?; // Mark notes as spent and remove them from the scanning cache for spend in &tx.sapling_spends { - wallet::sapling::mark_sapling_note_spent( - &up.wallet_db.conn, - tx_row, - spend.nf(), - )?; + wallet::sapling::mark_sapling_note_spent(&wdb.conn.0, tx_row, spend.nf())?; } for output in &tx.sapling_outputs { let received_note_id = - wallet::sapling::put_received_note(&up.wallet_db.conn, output, tx_row)?; + wallet::sapling::put_received_note(&wdb.conn.0, output, tx_row)?; // Save witness for note. new_witnesses.push((received_note_id, output.witness().clone())); @@ -560,17 +421,22 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { for (received_note_id, witness) in updated_witnesses.iter().chain(new_witnesses.iter()) { if let NoteId::ReceivedNoteId(rnid) = *received_note_id { - wallet::sapling::insert_witness(up, rnid, witness, block.block_height)?; + wallet::sapling::insert_witness( + &wdb.conn.0, + rnid, + witness, + block.block_height, + )?; } else { return Err(SqliteClientError::InvalidNoteId); } } // Prune the stored witnesses (we only expect rollbacks of at most PRUNING_HEIGHT blocks). - wallet::prune_witnesses(up, block.block_height - PRUNING_HEIGHT)?; + wallet::prune_witnesses(&wdb.conn.0, block.block_height - PRUNING_HEIGHT)?; // Update now-expired transactions that didn't get mined. - wallet::update_expired_notes(up, block.block_height)?; + wallet::update_expired_notes(&wdb.conn.0, block.block_height)?; Ok(new_witnesses) }) @@ -580,93 +446,114 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { &mut self, d_tx: DecryptedTransaction, ) -> Result { - self.transactionally(|up| { - let tx_ref = wallet::put_tx_data(up, d_tx.tx, None, None)?; - - let mut spending_account_id: Option = None; - for output in d_tx.sapling_outputs { - match output.transfer_type { - TransferType::Outgoing | TransferType::WalletInternal => { - let recipient = if output.transfer_type == TransferType::Outgoing { - Recipient::Sapling(output.note.recipient()) - } else { - Recipient::InternalAccount(output.account, PoolType::Sapling) - }; - - wallet::put_sent_output( - &up.wallet_db.conn, - &up.wallet_db.params, - output.account, - tx_ref, - output.index, - &recipient, - Amount::from_u64(output.note.value().inner()).map_err(|_| - SqliteClientError::CorruptedData("Note value is not a valid Zcash amount.".to_string()))?, - Some(&output.memo), - )?; + self.transactionally(|wdb| { + let tx_ref = wallet::put_tx_data(&wdb.conn.0, d_tx.tx, None, None)?; + + let mut spending_account_id: Option = None; + for output in d_tx.sapling_outputs { + match output.transfer_type { + TransferType::Outgoing | TransferType::WalletInternal => { + let recipient = if output.transfer_type == TransferType::Outgoing { + Recipient::Sapling(output.note.recipient()) + } else { + Recipient::InternalAccount(output.account, PoolType::Sapling) + }; + + wallet::put_sent_output( + &wdb.conn.0, + &wdb.params, + output.account, + tx_ref, + output.index, + &recipient, + Amount::from_u64(output.note.value().inner()).map_err(|_| { + SqliteClientError::CorruptedData( + "Note value is not a valid Zcash amount.".to_string(), + ) + })?, + Some(&output.memo), + )?; - if matches!(recipient, Recipient::InternalAccount(_, _)) { - wallet::sapling::put_received_note(&up.wallet_db.conn, output, tx_ref)?; - } + if matches!(recipient, Recipient::InternalAccount(_, _)) { + wallet::sapling::put_received_note(&wdb.conn.0, output, tx_ref)?; } - TransferType::Incoming => { - match spending_account_id { - Some(id) => - if id != output.account { - panic!("Unable to determine a unique account identifier for z->t spend."); - } - None => { - spending_account_id = Some(output.account); + } + TransferType::Incoming => { + match spending_account_id { + Some(id) => { + if id != output.account { + panic!("Unable to determine a unique account identifier for z->t spend."); } } - - wallet::sapling::put_received_note(&up.wallet_db.conn, output, tx_ref)?; + None => { + spending_account_id = Some(output.account); + } } + + wallet::sapling::put_received_note(&wdb.conn.0, output, tx_ref)?; } } + } - // If any of the utxos spent in the transaction are ours, mark them as spent. - #[cfg(feature = "transparent-inputs")] - for txin in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vin.iter()) { - wallet::mark_transparent_utxo_spent(&up.wallet_db.conn, tx_ref, &txin.prevout)?; - } + // If any of the utxos spent in the transaction are ours, mark them as spent. + #[cfg(feature = "transparent-inputs")] + for txin in d_tx + .tx + .transparent_bundle() + .iter() + .flat_map(|b| b.vin.iter()) + { + wallet::mark_transparent_utxo_spent(&wdb.conn.0, tx_ref, &txin.prevout)?; + } - // If we have some transparent outputs: - if !d_tx.tx.transparent_bundle().iter().any(|b| b.vout.is_empty()) { - let nullifiers = self.wallet_db.get_sapling_nullifiers(data_api::NullifierQuery::All)?; - // If the transaction contains shielded spends from our wallet, we will store z->t - // transactions we observe in the same way they would be stored by - // create_spend_to_address. - if let Some((account_id, _)) = nullifiers.iter().find( - |(_, nf)| - d_tx.tx.sapling_bundle().iter().flat_map(|b| b.shielded_spends().iter()) - .any(|input| nf == input.nullifier()) - ) { - for (output_index, txout) in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vout.iter()).enumerate() { - if let Some(address) = txout.recipient_address() { - wallet::put_sent_output( - &up.wallet_db.conn, - &up.wallet_db.params, - *account_id, - tx_ref, - output_index, - &Recipient::Transparent(address), - txout.value, - None - )?; - } + // If we have some transparent outputs: + if !d_tx + .tx + .transparent_bundle() + .iter() + .any(|b| b.vout.is_empty()) + { + let nullifiers = wdb.get_sapling_nullifiers(data_api::NullifierQuery::All)?; + // If the transaction contains shielded spends from our wallet, we will store z->t + // transactions we observe in the same way they would be stored by + // create_spend_to_address. + if let Some((account_id, _)) = nullifiers.iter().find(|(_, nf)| { + d_tx.tx + .sapling_bundle() + .iter() + .flat_map(|b| b.shielded_spends().iter()) + .any(|input| nf == input.nullifier()) + }) { + for (output_index, txout) in d_tx + .tx + .transparent_bundle() + .iter() + .flat_map(|b| b.vout.iter()) + .enumerate() + { + if let Some(address) = txout.recipient_address() { + wallet::put_sent_output( + &wdb.conn.0, + &wdb.params, + *account_id, + tx_ref, + output_index, + &Recipient::Transparent(address), + txout.value, + None, + )?; } } } - Ok(tx_ref) + } + Ok(tx_ref) }) } fn store_sent_tx(&mut self, sent_tx: &SentTransaction) -> Result { - // Update the database atomically, to ensure the result is internally consistent. - self.transactionally(|up| { + self.transactionally(|wdb| { let tx_ref = wallet::put_tx_data( - up, + &wdb.conn.0, sent_tx.tx, Some(sent_tx.fee_amount), Some(sent_tx.created), @@ -683,7 +570,7 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { if let Some(bundle) = sent_tx.tx.sapling_bundle() { for spend in bundle.shielded_spends() { wallet::sapling::mark_sapling_note_spent( - &up.wallet_db.conn, + &wdb.conn.0, tx_ref, spend.nullifier(), )?; @@ -692,13 +579,13 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { #[cfg(feature = "transparent-inputs")] for utxo_outpoint in &sent_tx.utxos_spent { - wallet::mark_transparent_utxo_spent(&up.wallet_db.conn, tx_ref, utxo_outpoint)?; + wallet::mark_transparent_utxo_spent(&wdb.conn.0, tx_ref, utxo_outpoint)?; } for output in &sent_tx.outputs { wallet::insert_sent_output( - &up.wallet_db.conn, - &up.wallet_db.params, + &wdb.conn.0, + &wdb.params, tx_ref, sent_tx.account, output, @@ -706,7 +593,7 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { if let Some((account, note)) = output.sapling_change_to() { wallet::sapling::put_received_note( - &up.wallet_db.conn, + &wdb.conn.0, &DecryptedOutput { index: output.output_index(), note: note.clone(), @@ -727,7 +614,9 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { } fn truncate_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error> { - wallet::truncate_to_height(self.wallet_db, block_height) + self.transactionally(|wdb| { + wallet::truncate_to_height(&wdb.conn.0, &wdb.params, block_height) + }) } fn put_received_transparent_utxo( @@ -735,11 +624,7 @@ impl<'a, P: consensus::Parameters> WalletWrite for DataConnStmtCache<'a, P> { _output: &WalletTransparentOutput, ) -> Result { #[cfg(feature = "transparent-inputs")] - return wallet::put_received_transparent_utxo( - &self.wallet_db.conn, - &self.wallet_db.params, - _output, - ); + return wallet::put_received_transparent_utxo(&self.conn, &self.params, _output); #[cfg(not(feature = "transparent-inputs"))] panic!( @@ -1083,7 +968,7 @@ mod tests { #[cfg(test)] pub(crate) fn init_test_accounts_table( - db_data: &WalletDb, + db_data: &mut WalletDb, ) -> (DiversifiableFullViewingKey, Option) { let (ufvk, taddr) = init_test_accounts_table_ufvk(db_data); (ufvk.sapling().unwrap().clone(), taddr) @@ -1091,7 +976,7 @@ mod tests { #[cfg(test)] pub(crate) fn init_test_accounts_table_ufvk( - db_data: &WalletDb, + db_data: &mut WalletDb, ) -> (UnifiedFullViewingKey, Option) { let seed = [0u8; 32]; let account = AccountId::from(0); @@ -1318,13 +1203,12 @@ mod tests { let account = AccountId::from(0); init_wallet_db(&mut db_data, None).unwrap(); - let _ = init_test_accounts_table_ufvk(&db_data); + init_test_accounts_table_ufvk(&mut db_data); let current_addr = db_data.get_current_address(account).unwrap(); assert!(current_addr.is_some()); - let mut update_ops = db_data.get_update_ops().unwrap(); - let addr2 = update_ops.get_next_available_address(account).unwrap(); + let addr2 = db_data.get_next_available_address(account).unwrap(); assert!(addr2.is_some()); assert_ne!(current_addr, addr2); @@ -1349,7 +1233,7 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet. - let (ufvk, taddr) = init_test_accounts_table_ufvk(&db_data); + let (ufvk, taddr) = init_test_accounts_table_ufvk(&mut db_data); let taddr = taddr.unwrap(); let receivers = db_data.get_transparent_receivers(0.into()).unwrap(); diff --git a/zcash_client_sqlite/src/prepared.rs b/zcash_client_sqlite/src/prepared.rs deleted file mode 100644 index 1304b39dd5..0000000000 --- a/zcash_client_sqlite/src/prepared.rs +++ /dev/null @@ -1,95 +0,0 @@ -//! Prepared SQL statements used by the wallet. -//! -//! Some `rusqlite` crate APIs are only available on prepared statements; these are stored -//! inside the [`DataConnStmtCache`]. When adding a new prepared statement: -//! -//! - Add it as a private field of `DataConnStmtCache`. -//! - Build the statement in [`DataConnStmtCache::new`]. -//! - Add a crate-private helper method to `DataConnStmtCache` for running the statement. - -use rusqlite::{params, Statement}; -use zcash_primitives::{consensus::BlockHeight, merkle_tree::write_incremental_witness, sapling}; - -use crate::{error::SqliteClientError, NoteId, WalletDb}; - -/// The primary type used to implement [`WalletWrite`] for the SQLite database. -/// -/// A data structure that stores the SQLite prepared statements that are -/// required for the implementation of [`WalletWrite`] against the backing -/// store. -/// -/// [`WalletWrite`]: zcash_client_backend::data_api::WalletWrite -pub struct DataConnStmtCache<'a, P> { - pub(crate) wallet_db: &'a WalletDb

, - - stmt_insert_witness: Statement<'a>, - stmt_prune_witnesses: Statement<'a>, - stmt_update_expired: Statement<'a>, -} - -impl<'a, P> DataConnStmtCache<'a, P> { - pub(crate) fn new(wallet_db: &'a WalletDb

) -> Result { - Ok( - DataConnStmtCache { - wallet_db, - #[cfg(feature = "transparent-inputs")] - stmt_insert_witness: wallet_db.conn.prepare( - "INSERT INTO sapling_witnesses (note, block, witness) - VALUES (?, ?, ?)", - )?, - stmt_prune_witnesses: wallet_db.conn.prepare( - "DELETE FROM sapling_witnesses WHERE block < ?" - )?, - stmt_update_expired: wallet_db.conn.prepare( - "UPDATE sapling_received_notes SET spent = NULL WHERE EXISTS ( - SELECT id_tx FROM transactions - WHERE id_tx = sapling_received_notes.spent AND block IS NULL AND expiry_height < ? - )", - )?, - } - ) - } -} - -impl<'a, P> DataConnStmtCache<'a, P> { - /// Records the incremental witness for the specified note, as of the given block - /// height. - /// - /// Returns `SqliteClientError::InvalidNoteId` if the note ID is for a sent note. - pub(crate) fn stmt_insert_witness( - &mut self, - note_id: NoteId, - height: BlockHeight, - witness: &sapling::IncrementalWitness, - ) -> Result<(), SqliteClientError> { - let note_id = match note_id { - NoteId::ReceivedNoteId(note_id) => Ok(note_id), - NoteId::SentNoteId(_) => Err(SqliteClientError::InvalidNoteId), - }?; - - let mut encoded = Vec::new(); - write_incremental_witness(witness, &mut encoded).unwrap(); - - self.stmt_insert_witness - .execute(params![note_id, u32::from(height), encoded])?; - - Ok(()) - } - - /// Removes old incremental witnesses up to the given block height. - pub(crate) fn stmt_prune_witnesses( - &mut self, - below_height: BlockHeight, - ) -> Result<(), SqliteClientError> { - self.stmt_prune_witnesses - .execute([u32::from(below_height)])?; - Ok(()) - } - - /// Marks notes that have not been mined in transactions as expired, up to the given - /// block height. - pub fn stmt_update_expired(&mut self, height: BlockHeight) -> Result<(), SqliteClientError> { - self.stmt_update_expired.execute([u32::from(height)])?; - Ok(()) - } -} diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 884e672fb1..c82ae16ca4 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1,4 +1,4 @@ -//! Functions for querying information in the wdb database. +//! Functions for querying information in the wallet database. //! //! These functions should generally not be used directly; instead, //! their functionality is available via the [`WalletRead`] and @@ -64,7 +64,7 @@ //! wallet. //! - `memo` the shielded memo associated with the output, if any. -use rusqlite::{named_params, params, Connection, OptionalExtension, ToSql}; +use rusqlite::{self, named_params, params, OptionalExtension, ToSql}; use std::collections::HashMap; use std::convert::TryFrom; @@ -89,7 +89,7 @@ use zcash_client_backend::{ wallet::WalletTx, }; -use crate::{error::SqliteClientError, DataConnStmtCache, WalletDb, PRUNING_HEIGHT}; +use crate::{error::SqliteClientError, PRUNING_HEIGHT}; #[cfg(feature = "transparent-inputs")] use { @@ -115,28 +115,28 @@ pub(crate) fn pool_code(pool_type: PoolType) -> i64 { } } -pub(crate) fn get_max_account_id

( - wdb: &WalletDb

, +pub(crate) fn get_max_account_id( + conn: &rusqlite::Connection, ) -> Result, SqliteClientError> { // This returns the most recently generated address. - wdb.conn - .query_row("SELECT MAX(account) FROM accounts", [], |row| { - let account_id: Option = row.get(0)?; - Ok(account_id.map(AccountId::from)) - }) - .map_err(SqliteClientError::from) + conn.query_row("SELECT MAX(account) FROM accounts", [], |row| { + let account_id: Option = row.get(0)?; + Ok(account_id.map(AccountId::from)) + }) + .map_err(SqliteClientError::from) } pub(crate) fn add_account( - wdb: &WalletDb

, + conn: &rusqlite::Transaction, + params: &P, account: AccountId, key: &UnifiedFullViewingKey, ) -> Result<(), SqliteClientError> { - add_account_internal(&wdb.conn, &wdb.params, "accounts", account, key) + add_account_internal(conn, params, "accounts", account, key) } pub(crate) fn add_account_internal>( - conn: &rusqlite::Connection, + conn: &rusqlite::Transaction, network: &P, accounts_table: &'static str, account: AccountId, @@ -159,12 +159,12 @@ pub(crate) fn add_account_internal( - wdb: &WalletDb

, + conn: &rusqlite::Connection, + params: &P, account: AccountId, ) -> Result, SqliteClientError> { // This returns the most recently generated address. - let addr: Option<(String, Vec)> = wdb - .conn + let addr: Option<(String, Vec)> = conn .query_row( "SELECT address, diversifier_index_be FROM addresses WHERE account = :account @@ -181,7 +181,7 @@ pub(crate) fn get_current_address( })?; di_be.reverse(); - RecipientAddress::decode(&wdb.params, &addr_str) + RecipientAddress::decode(params, &addr_str) .ok_or_else(|| { SqliteClientError::CorruptedData("Not a valid Zcash recipient address".to_owned()) }) @@ -201,7 +201,7 @@ pub(crate) fn get_current_address( /// /// Returns the database row for the newly-inserted address. pub(crate) fn insert_address( - conn: &Connection, + conn: &rusqlite::Connection, params: &P, account: AccountId, mut diversifier_index: DiversifierIndex, @@ -236,8 +236,8 @@ pub(crate) fn insert_address( #[cfg(feature = "transparent-inputs")] pub(crate) fn get_transparent_receivers( + conn: &rusqlite::Connection, params: &P, - conn: &Connection, account: AccountId, ) -> Result, SqliteClientError> { let mut ret = HashMap::new(); @@ -288,7 +288,7 @@ pub(crate) fn get_transparent_receivers( #[cfg(feature = "transparent-inputs")] pub(crate) fn get_legacy_transparent_address( params: &P, - conn: &Connection, + conn: &rusqlite::Connection, account: AccountId, ) -> Result, SqliteClientError> { // Get the UFVK for the account. @@ -322,18 +322,18 @@ pub(crate) fn get_legacy_transparent_address( /// Returns the [`UnifiedFullViewingKey`]s for the wallet. pub(crate) fn get_unified_full_viewing_keys( - wdb: &WalletDb

, + conn: &rusqlite::Connection, + params: &P, ) -> Result, SqliteClientError> { // Fetch the UnifiedFullViewingKeys we are tracking - let mut stmt_fetch_accounts = wdb - .conn - .prepare("SELECT account, ufvk FROM accounts ORDER BY account ASC")?; + let mut stmt_fetch_accounts = + conn.prepare("SELECT account, ufvk FROM accounts ORDER BY account ASC")?; let rows = stmt_fetch_accounts.query_map([], |row| { let acct: u32 = row.get(0)?; let account = AccountId::from(acct); let ufvk_str: String = row.get(1)?; - let ufvk = UnifiedFullViewingKey::decode(&wdb.params, &ufvk_str) + let ufvk = UnifiedFullViewingKey::decode(params, &ufvk_str) .map_err(SqliteClientError::CorruptedData); Ok((account, ufvk)) @@ -351,20 +351,20 @@ pub(crate) fn get_unified_full_viewing_keys( /// Returns the account id corresponding to a given [`UnifiedFullViewingKey`], /// if any. pub(crate) fn get_account_for_ufvk( - wdb: &WalletDb

, + conn: &rusqlite::Connection, + params: &P, ufvk: &UnifiedFullViewingKey, ) -> Result, SqliteClientError> { - wdb.conn - .query_row( - "SELECT account FROM accounts WHERE ufvk = ?", - [&ufvk.encode(&wdb.params)], - |row| { - let acct: u32 = row.get(0)?; - Ok(AccountId::from(acct)) - }, - ) - .optional() - .map_err(SqliteClientError::from) + conn.query_row( + "SELECT account FROM accounts WHERE ufvk = ?", + [&ufvk.encode(params)], + |row| { + let acct: u32 = row.get(0)?; + Ok(AccountId::from(acct)) + }, + ) + .optional() + .map_err(SqliteClientError::from) } /// Checks whether the specified [`ExtendedFullViewingKey`] is valid and corresponds to the @@ -372,15 +372,15 @@ pub(crate) fn get_account_for_ufvk( /// /// [`ExtendedFullViewingKey`]: zcash_primitives::zip32::ExtendedFullViewingKey pub(crate) fn is_valid_account_extfvk( - wdb: &WalletDb

, + conn: &rusqlite::Connection, + params: &P, account: AccountId, extfvk: &ExtendedFullViewingKey, ) -> Result { - wdb.conn - .prepare("SELECT ufvk FROM accounts WHERE account = ?")? + conn.prepare("SELECT ufvk FROM accounts WHERE account = ?")? .query_row([u32::from(account).to_sql()?], |row| { row.get(0).map(|ufvk_str: String| { - UnifiedFullViewingKey::decode(&wdb.params, &ufvk_str) + UnifiedFullViewingKey::decode(params, &ufvk_str) .map_err(SqliteClientError::CorruptedData) }) }) @@ -406,11 +406,11 @@ pub(crate) fn is_valid_account_extfvk( /// caveat. Use [`get_balance_at`] where you need a more reliable indication of the /// wallet balance. #[cfg(test)] -pub(crate) fn get_balance

( - wdb: &WalletDb

, +pub(crate) fn get_balance( + conn: &rusqlite::Connection, account: AccountId, ) -> Result { - let balance = wdb.conn.query_row( + let balance = conn.query_row( "SELECT SUM(value) FROM sapling_received_notes INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx WHERE account = ? AND spent IS NULL AND transactions.block IS NOT NULL", @@ -429,12 +429,12 @@ pub(crate) fn get_balance

( /// Returns the verified balance for the account at the specified height, /// This may be used to obtain a balance that ignores notes that have been /// received so recently that they are not yet deemed spendable. -pub(crate) fn get_balance_at

( - wdb: &WalletDb

, +pub(crate) fn get_balance_at( + conn: &rusqlite::Connection, account: AccountId, anchor_height: BlockHeight, ) -> Result { - let balance = wdb.conn.query_row( + let balance = conn.query_row( "SELECT SUM(value) FROM sapling_received_notes INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx WHERE account = ? AND spent IS NULL AND transactions.block <= ?", @@ -454,11 +454,11 @@ pub(crate) fn get_balance_at

( /// /// The note is identified by its row index in the `sapling_received_notes` table within the wdb /// database. -pub(crate) fn get_received_memo

( - wdb: &WalletDb

, +pub(crate) fn get_received_memo( + conn: &rusqlite::Connection, id_note: i64, ) -> Result, SqliteClientError> { - let memo_bytes: Option> = wdb.conn.query_row( + let memo_bytes: Option> = conn.query_row( "SELECT memo FROM sapling_received_notes WHERE id_note = ?", [id_note], @@ -476,10 +476,11 @@ pub(crate) fn get_received_memo

( /// Looks up a transaction by its internal database identifier. pub(crate) fn get_transaction( - wdb: &WalletDb

, + conn: &rusqlite::Connection, + params: &P, id_tx: i64, ) -> Result { - let (tx_bytes, block_height): (Vec<_>, BlockHeight) = wdb.conn.query_row( + let (tx_bytes, block_height): (Vec<_>, BlockHeight) = conn.query_row( "SELECT raw, block FROM transactions WHERE id_tx = ?", [id_tx], @@ -489,22 +490,19 @@ pub(crate) fn get_transaction( }, )?; - Transaction::read( - &tx_bytes[..], - BranchId::for_height(&wdb.params, block_height), - ) - .map_err(SqliteClientError::from) + Transaction::read(&tx_bytes[..], BranchId::for_height(params, block_height)) + .map_err(SqliteClientError::from) } /// Returns the memo for a sent note. /// /// The note is identified by its row index in the `sent_notes` table within the wdb /// database. -pub(crate) fn get_sent_memo

( - wdb: &WalletDb

, +pub(crate) fn get_sent_memo( + conn: &rusqlite::Connection, id_note: i64, ) -> Result, SqliteClientError> { - let memo_bytes: Option> = wdb.conn.query_row( + let memo_bytes: Option> = conn.query_row( "SELECT memo FROM sent_notes WHERE id_note = ?", [id_note], @@ -521,74 +519,70 @@ pub(crate) fn get_sent_memo

( } /// Returns the minimum and maximum heights for blocks stored in the wallet database. -pub(crate) fn block_height_extrema

( - wdb: &WalletDb

, +pub(crate) fn block_height_extrema( + conn: &rusqlite::Connection, ) -> Result, rusqlite::Error> { - wdb.conn - .query_row("SELECT MIN(height), MAX(height) FROM blocks", [], |row| { - let min_height: u32 = row.get(0)?; - let max_height: u32 = row.get(1)?; - Ok(Some(( - BlockHeight::from(min_height), - BlockHeight::from(max_height), - ))) - }) - //.optional() doesn't work here because a failed aggregate function - //produces a runtime error, not an empty set of rows. - .or(Ok(None)) + conn.query_row("SELECT MIN(height), MAX(height) FROM blocks", [], |row| { + let min_height: u32 = row.get(0)?; + let max_height: u32 = row.get(1)?; + Ok(Some(( + BlockHeight::from(min_height), + BlockHeight::from(max_height), + ))) + }) + //.optional() doesn't work here because a failed aggregate function + //produces a runtime error, not an empty set of rows. + .or(Ok(None)) } /// Returns the block height at which the specified transaction was mined, /// if any. -pub(crate) fn get_tx_height

( - wdb: &WalletDb

, +pub(crate) fn get_tx_height( + conn: &rusqlite::Connection, txid: TxId, ) -> Result, rusqlite::Error> { - wdb.conn - .query_row( - "SELECT block FROM transactions WHERE txid = ?", - [txid.as_ref().to_vec()], - |row| row.get(0).map(u32::into), - ) - .optional() + conn.query_row( + "SELECT block FROM transactions WHERE txid = ?", + [txid.as_ref().to_vec()], + |row| row.get(0).map(u32::into), + ) + .optional() } /// Returns the block hash for the block at the specified height, /// if any. -pub(crate) fn get_block_hash

( - wdb: &WalletDb

, +pub(crate) fn get_block_hash( + conn: &rusqlite::Connection, block_height: BlockHeight, ) -> Result, rusqlite::Error> { - wdb.conn - .query_row( - "SELECT hash FROM blocks WHERE height = ?", - [u32::from(block_height)], - |row| { - let row_data = row.get::<_, Vec<_>>(0)?; - Ok(BlockHash::from_slice(&row_data)) - }, - ) - .optional() + conn.query_row( + "SELECT hash FROM blocks WHERE height = ?", + [u32::from(block_height)], + |row| { + let row_data = row.get::<_, Vec<_>>(0)?; + Ok(BlockHash::from_slice(&row_data)) + }, + ) + .optional() } /// Gets the height to which the database must be truncated if any truncation that would remove a /// number of blocks greater than the pruning height is attempted. -pub(crate) fn get_min_unspent_height

( - wdb: &WalletDb

, +pub(crate) fn get_min_unspent_height( + conn: &rusqlite::Connection, ) -> Result, SqliteClientError> { - wdb.conn - .query_row( - "SELECT MIN(tx.block) + conn.query_row( + "SELECT MIN(tx.block) FROM sapling_received_notes n JOIN transactions tx ON tx.id_tx = n.tx WHERE n.spent IS NULL", - [], - |row| { - row.get(0) - .map(|maybe_height: Option| maybe_height.map(|height| height.into())) - }, - ) - .map_err(SqliteClientError::from) + [], + |row| { + row.get(0) + .map(|maybe_height: Option| maybe_height.map(|height| height.into())) + }, + ) + .map_err(SqliteClientError::from) } /// Truncates the database to the given height. @@ -598,25 +592,23 @@ pub(crate) fn get_min_unspent_height

( /// /// This should only be executed inside a transactional context. pub(crate) fn truncate_to_height( - wdb: &WalletDb

, + conn: &rusqlite::Transaction, + params: &P, block_height: BlockHeight, ) -> Result<(), SqliteClientError> { - let sapling_activation_height = wdb - .params + let sapling_activation_height = params .activation_height(NetworkUpgrade::Sapling) .expect("Sapling activation height mutst be available."); // Recall where we synced up to previously. - let last_scanned_height = wdb - .conn - .query_row("SELECT MAX(height) FROM blocks", [], |row| { - row.get(0) - .map(|h: u32| h.into()) - .or_else(|_| Ok(sapling_activation_height - 1)) - })?; + let last_scanned_height = conn.query_row("SELECT MAX(height) FROM blocks", [], |row| { + row.get(0) + .map(|h: u32| h.into()) + .or_else(|_| Ok(sapling_activation_height - 1)) + })?; if block_height < last_scanned_height - PRUNING_HEIGHT { - if let Some(h) = get_min_unspent_height(wdb)? { + if let Some(h) = get_min_unspent_height(conn)? { if block_height > h { return Err(SqliteClientError::RequestedRewindInvalid(h, block_height)); } @@ -626,13 +618,13 @@ pub(crate) fn truncate_to_height( // nothing to do if we're deleting back down to the max height if block_height < last_scanned_height { // Decrement witnesses. - wdb.conn.execute( + conn.execute( "DELETE FROM sapling_witnesses WHERE block > ?", [u32::from(block_height)], )?; // Rewind received notes - wdb.conn.execute( + conn.execute( "DELETE FROM sapling_received_notes WHERE id_note IN ( SELECT rn.id_note @@ -649,19 +641,19 @@ pub(crate) fn truncate_to_height( // presence of stale sent notes that link to unmined transactions. // Rewind utxos - wdb.conn.execute( + conn.execute( "DELETE FROM utxos WHERE height > ?", [u32::from(block_height)], )?; // Un-mine transactions. - wdb.conn.execute( + conn.execute( "UPDATE transactions SET block = NULL, tx_index = NULL WHERE block IS NOT NULL AND block > ?", [u32::from(block_height)], )?; // Now that they aren't depended on, delete scanned blocks. - wdb.conn.execute( + conn.execute( "DELETE FROM blocks WHERE height > ?", [u32::from(block_height)], )?; @@ -675,12 +667,13 @@ pub(crate) fn truncate_to_height( /// height less than or equal to the provided `max_height`. #[cfg(feature = "transparent-inputs")] pub(crate) fn get_unspent_transparent_outputs( - wdb: &WalletDb

, + conn: &rusqlite::Connection, + params: &P, address: &TransparentAddress, max_height: BlockHeight, exclude: &[OutPoint], ) -> Result, SqliteClientError> { - let mut stmt_blocks = wdb.conn.prepare( + let mut stmt_blocks = conn.prepare( "SELECT u.prevout_txid, u.prevout_idx, u.script, u.value_zat, u.height, tx.block as block FROM utxos u @@ -691,7 +684,7 @@ pub(crate) fn get_unspent_transparent_outputs( AND tx.block IS NULL", )?; - let addr_str = address.encode(&wdb.params); + let addr_str = address.encode(params); let mut utxos = Vec::::new(); let mut rows = stmt_blocks.query(params![addr_str, u32::from(max_height)])?; @@ -737,11 +730,12 @@ pub(crate) fn get_unspent_transparent_outputs( /// the provided `max_height`. #[cfg(feature = "transparent-inputs")] pub(crate) fn get_transparent_balances( - wdb: &WalletDb

, + conn: &rusqlite::Connection, + params: &P, account: AccountId, max_height: BlockHeight, ) -> Result, SqliteClientError> { - let mut stmt_blocks = wdb.conn.prepare( + let mut stmt_blocks = conn.prepare( "SELECT u.address, SUM(u.value_zat) FROM utxos u LEFT OUTER JOIN transactions tx @@ -756,7 +750,7 @@ pub(crate) fn get_transparent_balances( let mut rows = stmt_blocks.query(params![u32::from(account), u32::from(max_height)])?; while let Some(row) = rows.next()? { let taddr_str: String = row.get(0)?; - let taddr = TransparentAddress::decode(&wdb.params, &taddr_str)?; + let taddr = TransparentAddress::decode(params, &taddr_str)?; let value = Amount::from_i64(row.get(1)?).unwrap(); res.insert(taddr, value); @@ -766,8 +760,8 @@ pub(crate) fn get_transparent_balances( } /// Inserts information about a scanned block into the database. -pub(crate) fn insert_block<'a, P>( - stmts: &mut DataConnStmtCache<'a, P>, +pub(crate) fn insert_block( + conn: &rusqlite::Connection, block_height: BlockHeight, block_hash: BlockHash, block_time: u32, @@ -776,7 +770,7 @@ pub(crate) fn insert_block<'a, P>( let mut encoded_tree = Vec::new(); write_commitment_tree(commitment_tree, &mut encoded_tree).unwrap(); - let mut stmt_insert_block = stmts.wallet_db.conn.prepare_cached( + let mut stmt_insert_block = conn.prepare_cached( "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (?, ?, ?, ?)", )?; @@ -793,13 +787,13 @@ pub(crate) fn insert_block<'a, P>( /// Inserts information about a mined transaction that was observed to /// contain a note related to this wallet into the database. -pub(crate) fn put_tx_meta<'a, P, N>( - stmts: &mut DataConnStmtCache<'a, P>, +pub(crate) fn put_tx_meta( + conn: &rusqlite::Connection, tx: &WalletTx, height: BlockHeight, ) -> Result { // It isn't there, so insert our transaction into the database. - let mut stmt_upsert_tx_meta = stmts.wallet_db.conn.prepare_cached( + let mut stmt_upsert_tx_meta = conn.prepare_cached( "INSERT INTO transactions (txid, block, tx_index) VALUES (:txid, :block, :tx_index) ON CONFLICT (txid) DO UPDATE @@ -820,13 +814,13 @@ pub(crate) fn put_tx_meta<'a, P, N>( } /// Inserts full transaction data into the database. -pub(crate) fn put_tx_data<'a, P>( - stmts: &mut DataConnStmtCache<'a, P>, +pub(crate) fn put_tx_data( + conn: &rusqlite::Connection, tx: &Transaction, fee: Option, created_at: Option, ) -> Result { - let mut stmt_upsert_tx_data = stmts.wallet_db.conn.prepare_cached( + let mut stmt_upsert_tx_data = conn.prepare_cached( "INSERT INTO transactions (txid, created, expiry_height, raw, fee) VALUES (:txid, :created_at, :expiry_height, :raw, :fee) ON CONFLICT (txid) DO UPDATE @@ -856,7 +850,7 @@ pub(crate) fn put_tx_data<'a, P>( /// Marks the given UTXO as having been spent. #[cfg(feature = "transparent-inputs")] pub(crate) fn mark_transparent_utxo_spent( - conn: &Connection, + conn: &rusqlite::Connection, tx_ref: i64, outpoint: &OutPoint, ) -> Result<(), SqliteClientError> { @@ -879,7 +873,7 @@ pub(crate) fn mark_transparent_utxo_spent( /// Adds the given received UTXO to the datastore. #[cfg(feature = "transparent-inputs")] pub(crate) fn put_received_transparent_utxo( - conn: &Connection, + conn: &rusqlite::Connection, params: &P, output: &WalletTransparentOutput, ) -> Result { @@ -920,7 +914,7 @@ pub(crate) fn put_received_transparent_utxo( #[cfg(feature = "transparent-inputs")] pub(crate) fn put_legacy_transparent_utxo( - conn: &Connection, + conn: &rusqlite::Connection, params: &P, output: &WalletTransparentOutput, received_by_account: AccountId, @@ -958,20 +952,30 @@ pub(crate) fn put_legacy_transparent_utxo( } /// Removes old incremental witnesses up to the given block height. -pub(crate) fn prune_witnesses

( - stmts: &mut DataConnStmtCache<'_, P>, +pub(crate) fn prune_witnesses( + conn: &rusqlite::Connection, below_height: BlockHeight, ) -> Result<(), SqliteClientError> { - stmts.stmt_prune_witnesses(below_height) + let mut stmt_prune_witnesses = + conn.prepare_cached("DELETE FROM sapling_witnesses WHERE block < ?")?; + stmt_prune_witnesses.execute([u32::from(below_height)])?; + Ok(()) } /// Marks notes that have not been mined in transactions /// as expired, up to the given block height. -pub(crate) fn update_expired_notes

( - stmts: &mut DataConnStmtCache<'_, P>, +pub(crate) fn update_expired_notes( + conn: &rusqlite::Connection, height: BlockHeight, ) -> Result<(), SqliteClientError> { - stmts.stmt_update_expired(height) + let mut stmt_update_expired = conn.prepare_cached( + "UPDATE sapling_received_notes SET spent = NULL WHERE EXISTS ( + SELECT id_tx FROM transactions + WHERE id_tx = sapling_received_notes.spent AND block IS NULL AND expiry_height < ? + )", + )?; + stmt_update_expired.execute([u32::from(height)])?; + Ok(()) } // A utility function for creation of parameters for use in `insert_sent_output` @@ -1001,7 +1005,7 @@ fn recipient_params( /// /// This is a crate-internal convenience method. pub(crate) fn insert_sent_output( - conn: &Connection, + conn: &rusqlite::Connection, params: &P, tx_ref: i64, from_account: AccountId, @@ -1037,8 +1041,8 @@ pub(crate) fn insert_sent_output( /// /// This is a crate-internal convenience method. #[allow(clippy::too_many_arguments)] -pub(crate) fn put_sent_output<'a, P: consensus::Parameters>( - conn: &Connection, +pub(crate) fn put_sent_output( + conn: &rusqlite::Connection, params: &P, from_account: AccountId, tx_ref: i64, @@ -1114,11 +1118,11 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet - tests::init_test_accounts_table(&db_data); + tests::init_test_accounts_table(&mut db_data); // The account should be empty assert_eq!( - get_balance(&db_data, AccountId::from(0)).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), Amount::zero() ); @@ -1126,9 +1130,12 @@ mod tests { assert_eq!(db_data.get_target_and_anchor_heights(10).unwrap(), None); // An invalid account has zero balance - assert_matches!(get_current_address(&db_data, AccountId::from(1)), Ok(None)); + assert_matches!( + get_current_address(&db_data.conn, &db_data.params, AccountId::from(1)), + Ok(None) + ); assert_eq!( - get_balance(&db_data, AccountId::from(0)).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), Amount::zero() ); } @@ -1141,9 +1148,8 @@ mod tests { init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet - let mut ops = db_data.get_update_ops().unwrap(); let seed = Secret::new([0u8; 32].to_vec()); - let (account_id, _usk) = ops.create_account(&seed).unwrap(); + let (account_id, _usk) = db_data.create_account(&seed).unwrap(); let uaddr = db_data.get_current_address(account_id).unwrap().unwrap(); let taddr = uaddr.transparent().unwrap(); @@ -1162,8 +1168,7 @@ mod tests { ) .unwrap(); - let res0 = - super::put_received_transparent_utxo(&ops.wallet_db.conn, &ops.wallet_db.params, &utxo); + let res0 = super::put_received_transparent_utxo(&db_data.conn, &db_data.params, &utxo); assert_matches!(res0, Ok(_)); // Change the mined height of the UTXO and upsert; we should get back @@ -1177,16 +1182,13 @@ mod tests { BlockHeight::from_u32(34567), ) .unwrap(); - let res1 = super::put_received_transparent_utxo( - &ops.wallet_db.conn, - &ops.wallet_db.params, - &utxo2, - ); + let res1 = super::put_received_transparent_utxo(&db_data.conn, &db_data.params, &utxo2); assert_matches!(res1, Ok(id) if id == res0.unwrap()); assert_matches!( super::get_unspent_transparent_outputs( - &db_data, + &db_data.conn, + &db_data.params, taddr, BlockHeight::from_u32(12345), &[] @@ -1196,7 +1198,8 @@ mod tests { assert_matches!( super::get_unspent_transparent_outputs( - &db_data, + &db_data.conn, + &db_data.params, taddr, BlockHeight::from_u32(34567), &[] @@ -1222,11 +1225,7 @@ mod tests { ) .unwrap(); - let res2 = super::put_received_transparent_utxo( - &ops.wallet_db.conn, - &ops.wallet_db.params, - &utxo2, - ); + let res2 = super::put_received_transparent_utxo(&db_data.conn, &db_data.params, &utxo2); assert_matches!(res2, Err(_)); } } diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 6650322915..7f5a60ccc4 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -111,14 +111,14 @@ impl std::error::Error for WalletMigrationError { // check for unspent transparent outputs whenever running initialization with a version of the // library *not* compiled with the `transparent-inputs` feature flag, and fail if any are present. pub fn init_wallet_db( - wdb: &mut WalletDb

, + wdb: &mut WalletDb, seed: Option>, ) -> Result<(), MigratorError> { init_wallet_db_internal(wdb, seed, &[]) } fn init_wallet_db_internal( - wdb: &mut WalletDb

, + wdb: &mut WalletDb, seed: Option>, target_migrations: &[Uuid], ) -> Result<(), MigratorError> { @@ -200,7 +200,7 @@ fn init_wallet_db_internal( /// let dfvk = extsk.to_diversifiable_full_viewing_key(); /// let ufvk = UnifiedFullViewingKey::new(None, Some(dfvk), None).unwrap(); /// let ufvks = HashMap::from([(account, ufvk)]); -/// init_accounts_table(&db_data, &ufvks).unwrap(); +/// init_accounts_table(&mut db_data, &ufvks).unwrap(); /// # } /// ``` /// @@ -208,29 +208,29 @@ fn init_wallet_db_internal( /// [`scan_cached_blocks`]: zcash_client_backend::data_api::chain::scan_cached_blocks /// [`create_spend_to_address`]: zcash_client_backend::data_api::wallet::create_spend_to_address pub fn init_accounts_table( - wdb: &WalletDb

, + wallet_db: &mut WalletDb, keys: &HashMap, ) -> Result<(), SqliteClientError> { - let mut empty_check = wdb.conn.prepare("SELECT * FROM accounts LIMIT 1")?; - if empty_check.exists([])? { - return Err(SqliteClientError::TableNotEmpty); - } + wallet_db.transactionally(|wdb| { + let mut empty_check = wdb.conn.0.prepare("SELECT * FROM accounts LIMIT 1")?; + if empty_check.exists([])? { + return Err(SqliteClientError::TableNotEmpty); + } - // Ensure that the account identifiers are sequential and begin at zero. - if let Some(account_id) = keys.keys().max() { - if usize::try_from(u32::from(*account_id)).unwrap() >= keys.len() { - return Err(SqliteClientError::AccountIdDiscontinuity); + // Ensure that the account identifiers are sequential and begin at zero. + if let Some(account_id) = keys.keys().max() { + if usize::try_from(u32::from(*account_id)).unwrap() >= keys.len() { + return Err(SqliteClientError::AccountIdDiscontinuity); + } } - } - // Insert accounts atomically - wdb.conn.execute("BEGIN IMMEDIATE", [])?; - for (account, key) in keys.iter() { - wallet::add_account(wdb, *account, key)?; - } - wdb.conn.execute("COMMIT", [])?; + // Insert accounts atomically + for (account, key) in keys.iter() { + wallet::add_account(&wdb.conn.0, &wdb.params, *account, key)?; + } - Ok(()) + Ok(()) + }) } /// Initialises the data database with the given block. @@ -262,33 +262,35 @@ pub fn init_accounts_table( /// let sapling_tree = &[]; /// /// let data_file = NamedTempFile::new().unwrap(); -/// let db = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); -/// init_blocks_table(&db, height, hash, time, sapling_tree); +/// let mut db = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); +/// init_blocks_table(&mut db, height, hash, time, sapling_tree); /// ``` -pub fn init_blocks_table

( - wdb: &WalletDb

, +pub fn init_blocks_table( + wallet_db: &mut WalletDb, height: BlockHeight, hash: BlockHash, time: u32, sapling_tree: &[u8], ) -> Result<(), SqliteClientError> { - let mut empty_check = wdb.conn.prepare("SELECT * FROM blocks LIMIT 1")?; - if empty_check.exists([])? { - return Err(SqliteClientError::TableNotEmpty); - } + wallet_db.transactionally(|wdb| { + let mut empty_check = wdb.conn.0.prepare("SELECT * FROM blocks LIMIT 1")?; + if empty_check.exists([])? { + return Err(SqliteClientError::TableNotEmpty); + } - wdb.conn.execute( - "INSERT INTO blocks (height, hash, time, sapling_tree) + wdb.conn.0.execute( + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (?, ?, ?, ?)", - [ - u32::from(height).to_sql()?, - hash.0.to_sql()?, - time.to_sql()?, - sapling_tree.to_sql()?, - ], - )?; - - Ok(()) + [ + u32::from(height).to_sql()?, + hash.0.to_sql()?, + time.to_sql()?, + sapling_tree.to_sql()?, + ], + )?; + + Ok(()) + }) } #[cfg(test)] @@ -606,7 +608,7 @@ mod tests { #[test] fn init_migrate_from_0_3_0() { fn init_0_3_0

( - wdb: &mut WalletDb

, + wdb: &mut WalletDb, extfvk: &ExtendedFullViewingKey, account: AccountId, ) -> Result<(), rusqlite::Error> { @@ -722,7 +724,7 @@ mod tests { #[test] fn init_migrate_from_autoshielding_poc() { fn init_autoshielding

( - wdb: &WalletDb

, + wdb: &mut WalletDb, extfvk: &ExtendedFullViewingKey, account: AccountId, ) -> Result<(), rusqlite::Error> { @@ -878,14 +880,14 @@ mod tests { let extfvk = secret_key.to_extended_full_viewing_key(); let data_file = NamedTempFile::new().unwrap(); let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); - init_autoshielding(&db_data, &extfvk, account).unwrap(); + init_autoshielding(&mut db_data, &extfvk, account).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(seed.to_vec()))).unwrap(); } #[test] fn init_migrate_from_main_pre_migrations() { fn init_main

( - wdb: &WalletDb

, + wdb: &mut WalletDb, ufvk: &UnifiedFullViewingKey, account: AccountId, ) -> Result<(), rusqlite::Error> { @@ -1025,7 +1027,12 @@ mod tests { let secret_key = UnifiedSpendingKey::from_seed(&tests::network(), &seed, account).unwrap(); let data_file = NamedTempFile::new().unwrap(); let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); - init_main(&db_data, &secret_key.to_unified_full_viewing_key(), account).unwrap(); + init_main( + &mut db_data, + &secret_key.to_unified_full_viewing_key(), + account, + ) + .unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(seed.to_vec()))).unwrap(); } @@ -1036,8 +1043,8 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // We can call the function as many times as we want with no data - init_accounts_table(&db_data, &HashMap::new()).unwrap(); - init_accounts_table(&db_data, &HashMap::new()).unwrap(); + init_accounts_table(&mut db_data, &HashMap::new()).unwrap(); + init_accounts_table(&mut db_data, &HashMap::new()).unwrap(); let seed = [0u8; 32]; let account = AccountId::from(0); @@ -1062,11 +1069,11 @@ mod tests { let ufvk = UnifiedFullViewingKey::new(Some(dfvk), None).unwrap(); let ufvks = HashMap::from([(account, ufvk)]); - init_accounts_table(&db_data, &ufvks).unwrap(); + init_accounts_table(&mut db_data, &ufvks).unwrap(); // Subsequent calls should return an error - init_accounts_table(&db_data, &HashMap::new()).unwrap_err(); - init_accounts_table(&db_data, &ufvks).unwrap_err(); + init_accounts_table(&mut db_data, &HashMap::new()).unwrap_err(); + init_accounts_table(&mut db_data, &ufvks).unwrap_err(); } #[test] @@ -1090,12 +1097,12 @@ mod tests { // should fail if we have a gap assert_matches!( - init_accounts_table(&db_data, &ufvks(&[0, 2])), + init_accounts_table(&mut db_data, &ufvks(&[0, 2])), Err(SqliteClientError::AccountIdDiscontinuity) ); // should succeed if there are no gaps - assert!(init_accounts_table(&db_data, &ufvks(&[0, 1, 2])).is_ok()); + assert!(init_accounts_table(&mut db_data, &ufvks(&[0, 1, 2])).is_ok()); } #[test] @@ -1106,7 +1113,7 @@ mod tests { // First call with data should initialise the blocks table init_blocks_table( - &db_data, + &mut db_data, BlockHeight::from(1u32), BlockHash([1; 32]), 1, @@ -1116,7 +1123,7 @@ mod tests { // Subsequent calls should return an error init_blocks_table( - &db_data, + &mut db_data, BlockHeight::from(2u32), BlockHash([2; 32]), 2, @@ -1139,7 +1146,7 @@ mod tests { let ufvk = usk.to_unified_full_viewing_key(); let expected_address = ufvk.sapling().unwrap().default_address().1; let ufvks = HashMap::from([(account_id, ufvk)]); - init_accounts_table(&db_data, &ufvks).unwrap(); + init_accounts_table(&mut db_data, &ufvks).unwrap(); // The account's address should be in the data DB let ua = db_data.get_current_address(AccountId::from(0)).unwrap(); @@ -1153,16 +1160,15 @@ mod tests { let mut db_data = WalletDb::for_path(data_file.path(), Network::MainNetwork).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); - let mut ops = db_data.get_update_ops().unwrap(); let seed = test_vectors::UNIFIED[0].root_seed; - let (account, _usk) = ops.create_account(&Secret::new(seed.to_vec())).unwrap(); + let (account, _usk) = db_data.create_account(&Secret::new(seed.to_vec())).unwrap(); assert_eq!(account, AccountId::from(0u32)); for tv in &test_vectors::UNIFIED[..3] { if let Some(RecipientAddress::Unified(tvua)) = RecipientAddress::decode(&Network::MainNetwork, tv.unified_addr) { - let (ua, di) = wallet::get_current_address(&db_data, account) + let (ua, di) = wallet::get_current_address(&db_data.conn, &db_data.params, account) .unwrap() .expect("create_account generated the first address"); assert_eq!(DiversifierIndex::from(tv.diversifier_index), di); @@ -1170,7 +1176,8 @@ mod tests { assert_eq!(tvua.sapling(), ua.sapling()); assert_eq!(tv.unified_addr, ua.encode(&Network::MainNetwork)); - ops.get_next_available_address(account) + db_data + .get_next_available_address(account) .unwrap() .expect("get_next_available_address generated an address"); } else { diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs index f658ae1030..cc3c61f6ad 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs @@ -67,7 +67,7 @@ impl RusqliteMigration for Migration

{ while let Some(row) = rows.next()? { let account: u32 = row.get(0)?; let taddrs = - get_transparent_receivers(&self._params, transaction, AccountId::from(account)) + get_transparent_receivers(transaction, &self._params, AccountId::from(account)) .map_err(|e| match e { SqliteClientError::DbError(e) => WalletMigrationError::DbError(e), SqliteClientError::CorruptedData(s) => { diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 9419231baf..6f07f92b53 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -6,7 +6,7 @@ use std::rc::Rc; use zcash_primitives::{ consensus::BlockHeight, memo::MemoBytes, - merkle_tree::{read_commitment_tree, read_incremental_witness}, + merkle_tree::{read_commitment_tree, read_incremental_witness, write_incremental_witness}, sapling::{self, Diversifier, Note, Nullifier, Rseed}, transaction::components::Amount, zip32::AccountId, @@ -17,7 +17,7 @@ use zcash_client_backend::{ DecryptedOutput, TransferType, }; -use crate::{error::SqliteClientError, DataConnStmtCache, NoteId, WalletDb}; +use crate::{error::SqliteClientError, NoteId}; /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedSaplingOutput { @@ -230,43 +230,42 @@ pub(crate) fn select_spendable_sapling_notes( /// Returns the commitment tree for the block at the specified height, /// if any. -pub(crate) fn get_sapling_commitment_tree

( - wdb: &WalletDb

, +pub(crate) fn get_sapling_commitment_tree( + conn: &Connection, block_height: BlockHeight, ) -> Result, SqliteClientError> { - wdb.conn - .query_row_and_then( - "SELECT sapling_tree FROM blocks WHERE height = ?", - [u32::from(block_height)], - |row| { - let row_data: Vec = row.get(0)?; - read_commitment_tree(&row_data[..]).map_err(|e| { - rusqlite::Error::FromSqlConversionFailure( - row_data.len(), - rusqlite::types::Type::Blob, - Box::new(e), - ) - }) - }, - ) - .optional() - .map_err(SqliteClientError::from) + conn.query_row_and_then( + "SELECT sapling_tree FROM blocks WHERE height = ?", + [u32::from(block_height)], + |row| { + let row_data: Vec = row.get(0)?; + read_commitment_tree(&row_data[..]).map_err(|e| { + rusqlite::Error::FromSqlConversionFailure( + row_data.len(), + rusqlite::types::Type::Blob, + Box::new(e), + ) + }) + }, + ) + .optional() + .map_err(SqliteClientError::from) } /// Returns the incremental witnesses for the block at the specified height, /// if any. -pub(crate) fn get_sapling_witnesses

( - wdb: &WalletDb

, +pub(crate) fn get_sapling_witnesses( + conn: &Connection, block_height: BlockHeight, ) -> Result, SqliteClientError> { - let mut stmt_fetch_witnesses = wdb - .conn - .prepare("SELECT note, witness FROM sapling_witnesses WHERE block = ?")?; + let mut stmt_fetch_witnesses = + conn.prepare_cached("SELECT note, witness FROM sapling_witnesses WHERE block = ?")?; + let witnesses = stmt_fetch_witnesses .query_map([u32::from(block_height)], |row| { let id_note = NoteId::ReceivedNoteId(row.get(0)?); - let wdb: Vec = row.get(1)?; - Ok(read_incremental_witness(&wdb[..]).map(|witness| (id_note, witness))) + let witness_data: Vec = row.get(1)?; + Ok(read_incremental_witness(&witness_data[..]).map(|witness| (id_note, witness))) }) .map_err(SqliteClientError::from)?; @@ -277,13 +276,23 @@ pub(crate) fn get_sapling_witnesses

( /// Records the incremental witness for the specified note, /// as of the given block height. -pub(crate) fn insert_witness<'a, P>( - stmts: &mut DataConnStmtCache<'a, P>, +pub(crate) fn insert_witness( + conn: &Connection, note_id: i64, witness: &sapling::IncrementalWitness, height: BlockHeight, ) -> Result<(), SqliteClientError> { - stmts.stmt_insert_witness(NoteId::ReceivedNoteId(note_id), height, witness) + let mut stmt_insert_witness = conn.prepare_cached( + "INSERT INTO sapling_witnesses (note, block, witness) + VALUES (?, ?, ?)", + )?; + + let mut encoded = Vec::new(); + write_incremental_witness(witness, &mut encoded).unwrap(); + + stmt_insert_witness.execute(params![note_id, u32::from(height), encoded])?; + + Ok(()) } /// Retrieves the set of nullifiers for "potentially spendable" Sapling notes that the @@ -292,11 +301,11 @@ pub(crate) fn insert_witness<'a, P>( /// "Potentially spendable" means: /// - The transaction in which the note was created has been observed as mined. /// - No transaction in which the note's nullifier appears has been observed as mined. -pub(crate) fn get_sapling_nullifiers

( - wdb: &WalletDb

, +pub(crate) fn get_sapling_nullifiers( + conn: &Connection, ) -> Result, SqliteClientError> { // Get the nullifiers for the notes we are tracking - let mut stmt_fetch_nullifiers = wdb.conn.prepare( + let mut stmt_fetch_nullifiers = conn.prepare( "SELECT rn.id_note, rn.account, rn.nf, tx.block as block FROM sapling_received_notes rn LEFT OUTER JOIN transactions tx @@ -454,7 +463,7 @@ mod tests { get_balance, get_balance_at, init::{init_blocks_table, init_wallet_db}, }, - AccountId, BlockDb, DataConnStmtCache, WalletDb, + AccountId, BlockDb, WalletDb, }; #[cfg(feature = "transparent-inputs")] @@ -488,9 +497,8 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet - let mut ops = db_data.get_update_ops().unwrap(); let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = ops.create_account(&seed).unwrap(); + let (_, usk) = db_data.create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); let to = dfvk.default_address().1.into(); @@ -499,10 +507,9 @@ mod tests { let usk1 = UnifiedSpendingKey::from_seed(&network(), &[1u8; 32], acct1).unwrap(); // Attempting to spend with a USK that is not in the wallet results in an error - let mut db_write = db_data.get_update_ops().unwrap(); assert_matches!( create_spend_to_address( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &usk1, @@ -523,17 +530,15 @@ mod tests { init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet - let mut ops = db_data.get_update_ops().unwrap(); let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = ops.create_account(&seed).unwrap(); + let (_, usk) = db_data.create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); let to = dfvk.default_address().1.into(); // We cannot do anything if we aren't synchronised - let mut db_write = db_data.get_update_ops().unwrap(); assert_matches!( create_spend_to_address( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &usk, @@ -553,7 +558,7 @@ mod tests { let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); init_blocks_table( - &db_data, + &mut db_data, BlockHeight::from(1u32), BlockHash([1; 32]), 1, @@ -562,23 +567,21 @@ mod tests { .unwrap(); // Add an account to the wallet - let mut ops = db_data.get_update_ops().unwrap(); let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = ops.create_account(&seed).unwrap(); + let (_, usk) = db_data.create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); let to = dfvk.default_address().1.into(); // Account balance should be zero assert_eq!( - get_balance(&db_data, AccountId::from(0)).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), Amount::zero() ); // We cannot spend anything - let mut db_write = db_data.get_update_ops().unwrap(); assert_matches!( create_spend_to_address( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &usk, @@ -607,9 +610,8 @@ mod tests { init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet - let mut ops = db_data.get_update_ops().unwrap(); let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = ops.create_account(&seed).unwrap(); + let (_, usk) = db_data.create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note @@ -622,14 +624,16 @@ mod tests { value, ); insert_into_cache(&db_cache, &cb); - let mut db_write = db_data.get_update_ops().unwrap(); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); - assert_eq!(get_balance(&db_data, AccountId::from(0)).unwrap(), value); assert_eq!( - get_balance_at(&db_data, AccountId::from(0), anchor_height).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + value + ); + assert_eq!( + get_balance_at(&db_data.conn, AccountId::from(0), anchor_height).unwrap(), value ); @@ -642,16 +646,16 @@ mod tests { value, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Verified balance does not include the second note let (_, anchor_height2) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); assert_eq!( - get_balance(&db_data, AccountId::from(0)).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), (value + value).unwrap() ); assert_eq!( - get_balance_at(&db_data, AccountId::from(0), anchor_height2).unwrap(), + get_balance_at(&db_data.conn, AccountId::from(0), anchor_height2).unwrap(), value ); @@ -660,7 +664,7 @@ mod tests { let to = extsk2.default_address().1.into(); assert_matches!( create_spend_to_address( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &usk, @@ -690,12 +694,12 @@ mod tests { ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Second spend still fails assert_matches!( create_spend_to_address( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &usk, @@ -722,12 +726,12 @@ mod tests { value, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Second spend should now succeed assert_matches!( create_spend_to_address( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &usk, @@ -752,9 +756,8 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet - let mut ops = db_data.get_update_ops().unwrap(); let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = ops.create_account(&seed).unwrap(); + let (_, usk) = db_data.create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note @@ -767,16 +770,18 @@ mod tests { value, ); insert_into_cache(&db_cache, &cb); - let mut db_write = db_data.get_update_ops().unwrap(); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); - assert_eq!(get_balance(&db_data, AccountId::from(0)).unwrap(), value); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + assert_eq!( + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + value + ); // Send some of the funds to another address let extsk2 = ExtendedSpendingKey::master(&[]); let to = extsk2.default_address().1.into(); assert_matches!( create_spend_to_address( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &usk, @@ -792,7 +797,7 @@ mod tests { // A second spend fails because there are no usable notes assert_matches!( create_spend_to_address( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &usk, @@ -821,12 +826,12 @@ mod tests { ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Second spend still fails assert_matches!( create_spend_to_address( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &usk, @@ -852,11 +857,11 @@ mod tests { value, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Second spend should now succeed create_spend_to_address( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &usk, @@ -881,9 +886,8 @@ mod tests { init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet - let mut ops = db_data.get_update_ops().unwrap(); let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = ops.create_account(&seed).unwrap(); + let (_, usk) = db_data.create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note @@ -896,17 +900,19 @@ mod tests { value, ); insert_into_cache(&db_cache, &cb); - let mut db_write = db_data.get_update_ops().unwrap(); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); - assert_eq!(get_balance(&db_data, AccountId::from(0)).unwrap(), value); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + assert_eq!( + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + value + ); let extsk2 = ExtendedSpendingKey::master(&[]); let addr2 = extsk2.default_address().1; let to = addr2.into(); - let send_and_recover_with_policy = |db_write: &mut DataConnStmtCache<'_, _>, ovk_policy| { + let send_and_recover_with_policy = |db_data: &mut WalletDb, ovk_policy| { let tx_row = create_spend_to_address( - db_write, + db_data, &tests::network(), test_prover(), &usk, @@ -919,8 +925,7 @@ mod tests { .unwrap(); // Fetch the transaction from the database - let raw_tx: Vec<_> = db_write - .wallet_db + let raw_tx: Vec<_> = db_data .conn .query_row( "SELECT raw FROM transactions @@ -951,7 +956,7 @@ mod tests { // Send some of the funds to another address, keeping history. // The recipient output is decryptable by the sender. let (_, recovered_to, _) = - send_and_recover_with_policy(&mut db_write, OvkPolicy::Sender).unwrap(); + send_and_recover_with_policy(&mut db_data, OvkPolicy::Sender).unwrap(); assert_eq!(&recovered_to, &addr2); // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 42 (that don't send us funds) @@ -966,11 +971,11 @@ mod tests { ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&network, &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&network, &db_cache, &mut db_data, None).unwrap(); // Send the funds again, discarding history. // Neither transaction output is decryptable by the sender. - assert!(send_and_recover_with_policy(&mut db_write, OvkPolicy::Discard).is_none()); + assert!(send_and_recover_with_policy(&mut db_data, OvkPolicy::Discard).is_none()); } #[test] @@ -984,9 +989,8 @@ mod tests { init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet - let mut ops = db_data.get_update_ops().unwrap(); let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = ops.create_account(&seed).unwrap(); + let (_, usk) = db_data.create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note @@ -999,21 +1003,23 @@ mod tests { value, ); insert_into_cache(&db_cache, &cb); - let mut db_write = db_data.get_update_ops().unwrap(); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); - assert_eq!(get_balance(&db_data, AccountId::from(0)).unwrap(), value); assert_eq!( - get_balance_at(&db_data, AccountId::from(0), anchor_height).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + value + ); + assert_eq!( + get_balance_at(&db_data.conn, AccountId::from(0), anchor_height).unwrap(), value ); let to = TransparentAddress::PublicKey([7; 20]).into(); assert_matches!( create_spend_to_address( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &usk, @@ -1038,9 +1044,8 @@ mod tests { init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet - let mut ops = db_data.get_update_ops().unwrap(); let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = ops.create_account(&seed).unwrap(); + let (_, usk) = db_data.create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note @@ -1053,21 +1058,23 @@ mod tests { value, ); insert_into_cache(&db_cache, &cb); - let mut db_write = db_data.get_update_ops().unwrap(); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); - assert_eq!(get_balance(&db_data, AccountId::from(0)).unwrap(), value); assert_eq!( - get_balance_at(&db_data, AccountId::from(0), anchor_height).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + value + ); + assert_eq!( + get_balance_at(&db_data.conn, AccountId::from(0), anchor_height).unwrap(), value ); let to = TransparentAddress::PublicKey([7; 20]).into(); assert_matches!( create_spend_to_address( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &usk, @@ -1092,9 +1099,8 @@ mod tests { init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet - let mut ops = db_data.get_update_ops().unwrap(); let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = ops.create_account(&seed).unwrap(); + let (_, usk) = db_data.create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet @@ -1119,15 +1125,17 @@ mod tests { insert_into_cache(&db_cache, &cb); } - let mut db_write = db_data.get_update_ops().unwrap(); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); // Verified balance matches total balance let total = Amount::from_u64(60000).unwrap(); let (_, anchor_height) = db_data.get_target_and_anchor_heights(1).unwrap().unwrap(); - assert_eq!(get_balance(&db_data, AccountId::from(0)).unwrap(), total); assert_eq!( - get_balance_at(&db_data, AccountId::from(0), anchor_height).unwrap(), + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + total + ); + assert_eq!( + get_balance_at(&db_data.conn, AccountId::from(0), anchor_height).unwrap(), total ); @@ -1149,7 +1157,7 @@ mod tests { assert_matches!( spend( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &input_selector, @@ -1177,7 +1185,7 @@ mod tests { assert_matches!( spend( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &input_selector, @@ -1202,9 +1210,8 @@ mod tests { init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet - let mut db_write = db_data.get_update_ops().unwrap(); let seed = Secret::new([0u8; 32].to_vec()); - let (account_id, usk) = db_write.create_account(&seed).unwrap(); + let (account_id, usk) = db_data.create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); let uaddr = db_data.get_current_address(account_id).unwrap().unwrap(); let taddr = uaddr.transparent().unwrap(); @@ -1219,7 +1226,7 @@ mod tests { ) .unwrap(); - let res0 = db_write.put_received_transparent_utxo(&utxo); + let res0 = db_data.put_received_transparent_utxo(&utxo); assert!(matches!(res0, Ok(_))); let input_selector = GreedyInputSelector::new( @@ -1236,11 +1243,11 @@ mod tests { Amount::from_u64(50000).unwrap(), ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_write, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); assert_matches!( shield_transparent_funds( - &mut db_write, + &mut db_data, &tests::network(), test_prover(), &input_selector, From 4b9180cb04c538ba9a76fbf63f3003bdea5d1997 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 12 Jun 2023 12:17:20 -0600 Subject: [PATCH 0009/1122] zcash_client_sqlite: Fix details of `put_sent_output` documentation. Also address a minor naming issue from code review. --- zcash_client_sqlite/src/lib.rs | 8 ++++---- zcash_client_sqlite/src/wallet.rs | 23 ++++++++++------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index e3b13daf34..1835b27a9c 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -108,9 +108,9 @@ pub struct WalletDb { } /// A wrapper for a SQLite transaction affecting the wallet database. -pub struct WalletTransaction<'conn>(pub(crate) rusqlite::Transaction<'conn>); +pub struct SqlTransaction<'conn>(pub(crate) rusqlite::Transaction<'conn>); -impl Borrow for WalletTransaction<'_> { +impl Borrow for SqlTransaction<'_> { fn borrow(&self) -> &rusqlite::Connection { &self.0 } @@ -127,10 +127,10 @@ impl WalletDb { pub fn transactionally(&mut self, f: F) -> Result where - F: FnOnce(&WalletDb, P>) -> Result, + F: FnOnce(&WalletDb, P>) -> Result, { let wdb = WalletDb { - conn: WalletTransaction(self.conn.transaction()?), + conn: SqlTransaction(self.conn.transaction()?), params: self.params.clone(), }; let result = f(&wdb)?; diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index c82ae16ca4..b1cd5998f3 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -980,15 +980,6 @@ pub(crate) fn update_expired_notes( // A utility function for creation of parameters for use in `insert_sent_output` // and `put_sent_output` -// -// - If `to` is a Unified address, this is an index into the outputs of the transaction -// within the bundle associated with the recipient's output pool. -// - If `to` is a Sapling address, this is an index into the Sapling outputs of the -// transaction. -// - If `to` is a transparent address, this is an index into the transparent outputs of -// the transaction. -// - If `to` is an internal account, this is an index into the Sapling outputs of the -// transaction. fn recipient_params( params: &P, to: &Recipient, @@ -1002,8 +993,6 @@ fn recipient_params( } /// Records information about a transaction output that your wallet created. -/// -/// This is a crate-internal convenience method. pub(crate) fn insert_sent_output( conn: &rusqlite::Connection, params: &P, @@ -1037,9 +1026,17 @@ pub(crate) fn insert_sent_output( Ok(()) } -/// Records information about a transaction output that your wallet created. +/// Records information about a transaction output that your wallet created, from the constituent +/// properties of that output. /// -/// This is a crate-internal convenience method. +/// - If `recipient` is a Unified address, `output_index` is an index into the outputs of the +/// transaction within the bundle associated with the recipient's output pool. +/// - If `recipient` is a Sapling address, `output_index` is an index into the Sapling outputs of +/// the transaction. +/// - If `recipient` is a transparent address, `output_index` is an index into the transparent +/// outputs of the transaction. +/// - If `recipient` is an internal account, `output_index` is an index into the Sapling outputs of +/// the transaction. #[allow(clippy::too_many_arguments)] pub(crate) fn put_sent_output( conn: &rusqlite::Connection, From 820e61caaa7c4b4c825c11da8f975169ae2c0729 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 16 Jun 2023 15:06:32 -0600 Subject: [PATCH 0010/1122] zcash_client_sqlite: Avoid swallowed errors in max-block-height queries. --- zcash_client_sqlite/src/wallet.rs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index b1cd5998f3..23f4c965e0 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -523,16 +523,12 @@ pub(crate) fn block_height_extrema( conn: &rusqlite::Connection, ) -> Result, rusqlite::Error> { conn.query_row("SELECT MIN(height), MAX(height) FROM blocks", [], |row| { - let min_height: u32 = row.get(0)?; - let max_height: u32 = row.get(1)?; - Ok(Some(( - BlockHeight::from(min_height), - BlockHeight::from(max_height), - ))) + let min_height: Option = row.get(0)?; + let max_height: Option = row.get(1)?; + Ok(min_height + .map(BlockHeight::from) + .zip(max_height.map(BlockHeight::from))) }) - //.optional() doesn't work here because a failed aggregate function - //produces a runtime error, not an empty set of rows. - .or(Ok(None)) } /// Returns the block height at which the specified transaction was mined, @@ -602,9 +598,8 @@ pub(crate) fn truncate_to_height( // Recall where we synced up to previously. let last_scanned_height = conn.query_row("SELECT MAX(height) FROM blocks", [], |row| { - row.get(0) - .map(|h: u32| h.into()) - .or_else(|_| Ok(sapling_activation_height - 1)) + row.get::<_, Option>(0) + .map(|opt| opt.map_or_else(|| sapling_activation_height - 1, BlockHeight::from)) })?; if block_height < last_scanned_height - PRUNING_HEIGHT { From 91f0f03d9d5c4cf78b9f8e68e5794d4abcb93c5d Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 16 Jun 2023 15:07:29 -0600 Subject: [PATCH 0011/1122] zcash_client_sqlite: Factor out common logic for the representation of stored memos. --- zcash_client_sqlite/src/wallet.rs | 7 ++++++- zcash_client_sqlite/src/wallet/sapling.rs | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 23f4c965e0..3f5515e2d8 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -115,6 +115,11 @@ pub(crate) fn pool_code(pool_type: PoolType) -> i64 { } } +pub(crate) fn memo_repr(memo: Option<&MemoBytes>) -> Option<&[u8]> { + memo.filter(|m| *m != &MemoBytes::empty()) + .map(|m| m.as_slice()) +} + pub(crate) fn get_max_account_id( conn: &rusqlite::Connection, ) -> Result, SqliteClientError> { @@ -1067,7 +1072,7 @@ pub(crate) fn put_sent_output( ":to_address": &to_address, ":to_account": &to_account, ":value": &i64::from(value), - ":memo": &memo.filter(|m| *m != &MemoBytes::empty()).map(|m| m.as_slice()), + ":memo": memo_repr(memo) ]; stmt_upsert_sent_output.execute(sql_args)?; diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 6f07f92b53..66734cfdc3 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -19,6 +19,8 @@ use zcash_client_backend::{ use crate::{error::SqliteClientError, NoteId}; +use super::memo_repr; + /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedSaplingOutput { fn index(&self) -> usize; @@ -407,9 +409,7 @@ pub(crate) fn put_received_note( ":value": output.note().value().inner(), ":rcm": &rcm.as_ref(), ":nf": output.nullifier().map(|nf| nf.0.as_ref()), - ":memo": output.memo() - .filter(|m| *m != &MemoBytes::empty()) - .map(|m| m.as_slice()), + ":memo": memo_repr(output.memo()), ":is_change": output.is_change() ]; From 48434bb271c329c9767382d948f5c525ec017e15 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 16 Jun 2023 15:07:53 -0600 Subject: [PATCH 0012/1122] zcash_client_sqlite: Fix SQL identation errors. --- zcash_client_sqlite/src/wallet.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 3f5515e2d8..a05597134f 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -626,13 +626,13 @@ pub(crate) fn truncate_to_height( // Rewind received notes conn.execute( "DELETE FROM sapling_received_notes - WHERE id_note IN ( - SELECT rn.id_note - FROM sapling_received_notes rn - LEFT OUTER JOIN transactions tx - ON tx.id_tx = rn.tx - WHERE tx.block IS NOT NULL AND tx.block > ? - );", + WHERE id_note IN ( + SELECT rn.id_note + FROM sapling_received_notes rn + LEFT OUTER JOIN transactions tx + ON tx.id_tx = rn.tx + WHERE tx.block IS NOT NULL AND tx.block > ? + );", [u32::from(block_height)], )?; @@ -969,11 +969,11 @@ pub(crate) fn update_expired_notes( height: BlockHeight, ) -> Result<(), SqliteClientError> { let mut stmt_update_expired = conn.prepare_cached( - "UPDATE sapling_received_notes SET spent = NULL WHERE EXISTS ( - SELECT id_tx FROM transactions - WHERE id_tx = sapling_received_notes.spent AND block IS NULL AND expiry_height < ? - )", - )?; + "UPDATE sapling_received_notes SET spent = NULL WHERE EXISTS ( + SELECT id_tx FROM transactions + WHERE id_tx = sapling_received_notes.spent AND block IS NULL AND expiry_height < ? + )", + )?; stmt_update_expired.execute([u32::from(height)])?; Ok(()) } From 5ba44b901970da2134bd5bac6d28e9b55ede0a4d Mon Sep 17 00:00:00 2001 From: Nate Wilcox Date: Sat, 17 Jun 2023 12:44:35 -0700 Subject: [PATCH 0013/1122] Update dependencies to resolve https://rustsec.org/advisories/RUSTSEC-2022-0090 --- zcash_client_backend/Cargo.toml | 2 +- zcash_client_sqlite/Cargo.toml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index bcd0038e33..eb6eb2c1d9 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -29,7 +29,7 @@ zcash_primitives = { version = "0.12", path = "../zcash_primitives", default-fea # Dependencies exposed in a public API: # (Breaking upgrades to these require a breaking upgrade to this crate.) # - Data Access API -time = "0.2" +time = "0.3.22" # - Encodings base64 = "0.21" diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index c58dba2a7d..dacaa00c7f 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -36,10 +36,10 @@ secrecy = "0.8" # - SQLite databases group = "0.13" jubjub = "0.10" -rusqlite = { version = "0.25", features = ["bundled", "time", "array"] } +rusqlite = { version = "0.29.0", features = ["bundled", "time", "array"] } schemer = "0.2" -schemer-rusqlite = "0.2.1" -time = "0.2" +schemer-rusqlite = "0.2.2" +time = "0.3.22" uuid = "1.1" # Dependencies used internally: From dce8676974556a453df991691356063b7223e4d6 Mon Sep 17 00:00:00 2001 From: Hazel OHearn Date: Tue, 15 Nov 2022 15:57:13 -0400 Subject: [PATCH 0014/1122] Remove impls of `{PartialEq, Eq}` for `zcash_primitives::transaction::builder::Error` Co-authored-by: Jack Grigg --- zcash_primitives/CHANGELOG.md | 8 +++++-- zcash_primitives/src/transaction/builder.rs | 25 ++++++++++----------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 4f3d765178..23dbfbd048 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -7,6 +7,10 @@ and this library adheres to Rust's notion of ## [Unreleased] +### Removed +- Removed `impl {PartialEq, Eq} for transaction::builder::Error` in favor + of using `assert_matches!` where error comparisons are required. + ## [0.12.0] - 2023-06-06 ### Added - `zcash_primitives::transaction`: @@ -27,7 +31,7 @@ and this library adheres to Rust's notion of `incrementalmerkletree::Hashable` and `merkle_tree::HashSer`. - The `Hashable` bound on the `Node` parameter to the `IncrementalWitness` type has been removed. -- `sapling::SAPLING_COMMITMENT_TREE_DEPTH_U8` and `sapling::SAPLING_COMMITMENT_TREE_DEPTH` +- `sapling::SAPLING_COMMITMENT_TREE_DEPTH_U8` and `sapling::SAPLING_COMMITMENT_TREE_DEPTH` have been removed; use `sapling::NOTE_COMMITMENT_TREE_DEPTH` instead. - `merkle_tree::{CommitmentTree, IncrementalWitness, MerklePath}` have been removed in favor of versions of these types that are now provided by the @@ -89,7 +93,7 @@ and this library adheres to Rust's notion of - The bounds on the `H` parameter to the following methods have changed: - `merkle_tree::incremental::read_frontier_v0` - `merkle_tree::incremental::read_auth_fragment_v1` -- The depth of the `merkle_tree::{CommitmentTree, IncrementalWitness, and MerklePath}` +- The depth of the `merkle_tree::{CommitmentTree, IncrementalWitness, and MerklePath}` data types are now statically constrained using const generic type parameters. - `transaction::fees::fixed::FeeRule::standard()` now uses the ZIP 317 minimum fee (10000 zatoshis rather than 1000 zatoshis) as the fixed fee. To be compliant with diff --git a/zcash_primitives/src/transaction/builder.rs b/zcash_primitives/src/transaction/builder.rs index 958a64454e..0b9a3a3ed0 100644 --- a/zcash_primitives/src/transaction/builder.rs +++ b/zcash_primitives/src/transaction/builder.rs @@ -50,7 +50,7 @@ use crate::{ const DEFAULT_TX_EXPIRY_DELTA: u32 = 40; /// Errors that can occur during transaction construction. -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug)] pub enum Error { /// Insufficient funds were provided to the transaction builder; the given /// additional amount is required in order to construct the transaction. @@ -543,6 +543,7 @@ mod testing { #[cfg(test)] mod tests { + use assert_matches::assert_matches; use ff::Field; use incrementalmerkletree::{frontier::CommitmentTree, witness::IncrementalWitness}; use rand_core::OsRng; @@ -691,7 +692,7 @@ mod tests { // Expect a binding signature error, because our inputs aren't valid, but this shows // that a binding signature was attempted - assert_eq!( + assert_matches!( builder.mock_build(), Err(Error::SaplingBuild(sapling_builder::Error::BindingSig)) ); @@ -728,7 +729,7 @@ mod tests { // 0.0001 t-ZEC fee { let builder = Builder::new(TEST_NETWORK, tx_height); - assert_eq!( + assert_matches!( builder.mock_build(), Err(Error::InsufficientFunds(MINIMUM_FEE)) ); @@ -750,11 +751,10 @@ mod tests { MemoBytes::empty(), ) .unwrap(); - assert_eq!( + assert_matches!( builder.mock_build(), - Err(Error::InsufficientFunds( - (Amount::from_i64(50000).unwrap() + MINIMUM_FEE).unwrap() - )) + Err(Error::InsufficientFunds(expected)) if + expected == (Amount::from_i64(50000).unwrap() + MINIMUM_FEE).unwrap() ); } @@ -768,11 +768,10 @@ mod tests { Amount::from_u64(50000).unwrap(), ) .unwrap(); - assert_eq!( + assert_matches!( builder.mock_build(), - Err(Error::InsufficientFunds( + Err(Error::InsufficientFunds(expected)) if expected == (Amount::from_i64(50000).unwrap() + MINIMUM_FEE).unwrap() - )) ); } @@ -808,9 +807,9 @@ mod tests { Amount::from_u64(20000).unwrap(), ) .unwrap(); - assert_eq!( + assert_matches!( builder.mock_build(), - Err(Error::InsufficientFunds(Amount::from_i64(1).unwrap())) + Err(Error::InsufficientFunds(expected)) if expected == Amount::from_i64(1).unwrap() ); } @@ -852,7 +851,7 @@ mod tests { Amount::from_u64(20000).unwrap(), ) .unwrap(); - assert_eq!( + assert_matches!( builder.mock_build(), Err(Error::SaplingBuild(sapling_builder::Error::BindingSig)) ) From 4fbdd64c89703053eeaeee51750ea6f5b93a3ec9 Mon Sep 17 00:00:00 2001 From: Hazel OHearn Date: Tue, 15 Nov 2022 15:57:13 -0400 Subject: [PATCH 0015/1122] Add Orchard support to `zcash_primitives::transaction::builder::Builder` Co-authored-by: Jack Grigg Co-authored-by: Kris Nuttycombe --- zcash_client_backend/src/data_api/wallet.rs | 2 +- zcash_client_backend/src/fees/fixed.rs | 2 + zcash_client_backend/src/fees/zip317.rs | 2 + zcash_client_backend/src/keys.rs | 2 + zcash_extensions/src/transparent/demo.rs | 2 +- zcash_primitives/CHANGELOG.md | 26 ++- zcash_primitives/src/consensus.rs | 2 +- zcash_primitives/src/transaction/builder.rs | 208 +++++++++++++++--- .../src/transaction/components/orchard.rs | 8 - zcash_primitives/src/transaction/fees.rs | 2 + .../src/transaction/fees/fixed.rs | 1 + .../src/transaction/fees/zip317.rs | 4 +- zcash_primitives/src/transaction/mod.rs | 3 +- 13 files changed, 223 insertions(+), 41 deletions(-) diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index d96a47e284..160529dc6a 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -459,7 +459,7 @@ where // Create the transaction. The type of the proposal ensures that there // are no possible transparent inputs, so we ignore those - let mut builder = Builder::new(params.clone(), proposal.target_height()); + let mut builder = Builder::new(params.clone(), proposal.target_height(), None); for selected in proposal.sapling_inputs() { let (note, key, merkle_path) = select_key_for_note(selected, usk.sapling(), &dfvk) diff --git a/zcash_client_backend/src/fees/fixed.rs b/zcash_client_backend/src/fees/fixed.rs index 28b309bb5a..59732f8382 100644 --- a/zcash_client_backend/src/fees/fixed.rs +++ b/zcash_client_backend/src/fees/fixed.rs @@ -78,6 +78,8 @@ impl ChangeStrategy for SingleOutputChangeStrategy { transparent_outputs, sapling_inputs.len(), sapling_outputs.len() + 1, + //Orchard is not yet supported in zcash_client_backend + 0, ) .unwrap(); // fixed::FeeRule::fee_required is infallible. diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index 142bf7ce65..6d2ddc62ab 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -167,6 +167,8 @@ impl ChangeStrategy for SingleOutputChangeStrategy { // add one for Sapling change, then account for Sapling output padding performed by // the transaction builder std::cmp::max(sapling_outputs.len() + 1, 2), + //Orchard is not yet supported in zcash_client_backend + 0, ) .map_err(ChangeError::StrategyError)?; diff --git a/zcash_client_backend/src/keys.rs b/zcash_client_backend/src/keys.rs index 54a585dcb4..d7480c74d6 100644 --- a/zcash_client_backend/src/keys.rs +++ b/zcash_client_backend/src/keys.rs @@ -703,6 +703,8 @@ mod tests { #[test] #[cfg(feature = "transparent-inputs")] fn ufvk_derivation() { + use super::UnifiedSpendingKey; + for tv in test_vectors::UNIFIED { let usk = UnifiedSpendingKey::from_seed( &MAIN_NETWORK, diff --git a/zcash_extensions/src/transparent/demo.rs b/zcash_extensions/src/transparent/demo.rs index b949817327..b5cd5e0be0 100644 --- a/zcash_extensions/src/transparent/demo.rs +++ b/zcash_extensions/src/transparent/demo.rs @@ -638,7 +638,7 @@ mod tests { fn demo_builder<'a>(height: BlockHeight) -> DemoBuilder> { DemoBuilder { - txn_builder: Builder::new(FutureNetwork, height), + txn_builder: Builder::new(FutureNetwork, height, None), extension_id: 0, } } diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 23dbfbd048..7e1c46ddfa 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -7,9 +7,31 @@ and this library adheres to Rust's notion of ## [Unreleased] +### Added +- `zcash_primitives::transaction::builder`: + - `Builder::add_orchard_spend` + - `Builder::add_orchard_output` +- `zcash_primitives::transaction::components::orchard::builder` module + +### Changed +- `zcash_primitives::transaction`: + - `builder::Builder::{new, new_with_rng}` now takes an optional `orchard_anchor` + argument which must be provided in order to enable Orchard spends and recipients. + - `builder::Builder::test_only_new_with_rng` + now returns an existential type: `Builder<'a, P, impl RngCore + CryptoRng>` + instead of `Builder<'a, P, R>` + - `builder::Error` has several additional variants for Orchard-related errors. + - `fees::FeeRule::fee_required` now takes an additional argument, + `orchard_action_count` + - `Unauthorized`'s associated type `OrchardAuth` is now + `orchard::builder::InProgress` + instead of `zcash_primitives::transaction::components::orchard::Unauthorized` +- `zcash_primitives::consensus::NetworkUpgrade` now implements `PartialEq`, `Eq` + ### Removed -- Removed `impl {PartialEq, Eq} for transaction::builder::Error` in favor - of using `assert_matches!` where error comparisons are required. +- `impl {PartialEq, Eq} for transaction::builder::Error` + (use `assert_matches!` where error comparisons are required) +- `zcash_primitives::transaction::components::orchard::Unauthorized` ## [0.12.0] - 2023-06-06 ### Added diff --git a/zcash_primitives/src/consensus.rs b/zcash_primitives/src/consensus.rs index 02ceffa165..dc972f7008 100644 --- a/zcash_primitives/src/consensus.rs +++ b/zcash_primitives/src/consensus.rs @@ -361,7 +361,7 @@ impl Parameters for Network { /// consensus rules enforced by the network are altered. /// /// See [ZIP 200](https://zips.z.cash/zip-0200) for more details. -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum NetworkUpgrade { /// The [Overwinter] network upgrade. /// diff --git a/zcash_primitives/src/transaction/builder.rs b/zcash_primitives/src/transaction/builder.rs index 0b9a3a3ed0..953c1c6aac 100644 --- a/zcash_primitives/src/transaction/builder.rs +++ b/zcash_primitives/src/transaction/builder.rs @@ -8,7 +8,7 @@ use std::sync::mpsc::Sender; use rand::{rngs::OsRng, CryptoRng, RngCore}; use crate::{ - consensus::{self, BlockHeight, BranchId}, + consensus::{self, BlockHeight, BranchId, NetworkUpgrade}, keys::OutgoingViewingKey, legacy::TransparentAddress, memo::MemoBytes, @@ -66,6 +66,15 @@ pub enum Error { TransparentBuild(transparent::builder::Error), /// An error occurred in constructing the Sapling parts of a transaction. SaplingBuild(sapling_builder::Error), + /// An error occurred in constructing the Orchard parts of a transaction. + OrchardBuild(orchard::builder::BuildError), + /// An error occurred in adding an Orchard Spend a transaction. + OrchardSpend(orchard::builder::SpendError), + /// An error occurred in adding an Orchard Output a transaction. + OrchardRecipient(orchard::builder::OutputError), + /// Orchard is not yet active on the network, and the user attempted to add + /// Orchard parts to a transaction. + OrchardAnchorNotAvailable, /// An error occurred in constructing the TZE parts of a transaction. #[cfg(feature = "zfuture")] TzeBuild(tze::builder::Error), @@ -88,6 +97,13 @@ impl fmt::Display for Error { Error::Fee(e) => write!(f, "An error occurred in fee calculation: {}", e), Error::TransparentBuild(err) => err.fmt(f), Error::SaplingBuild(err) => err.fmt(f), + Error::OrchardBuild(err) => write!(f, "{:?}", err), + Error::OrchardSpend(err) => write!(f, "Could not add orchard spend: {}", err), + Error::OrchardRecipient(err) => write!(f, "Could not add orchard recipient: {}", err), + Error::OrchardAnchorNotAvailable => write!( + f, + "Cannot create orchard transactions without an Orchard anchor or before NU5 activation" + ), #[cfg(feature = "zfuture")] Error::TzeBuild(err) => err.fmt(f), } @@ -139,6 +155,12 @@ pub struct Builder<'a, P, R> { expiry_height: BlockHeight, transparent_builder: TransparentBuilder, sapling_builder: SaplingBuilder

, + orchard_builder: Option, + // TODO: In the future, instead of taking the spending keys as arguments when calling + // `add_sapling_spend` or `add_orchard_spend`, we will build an unauthorized, unproven + // transaction, and then the caller will be responsible for using the spending keys or their + // derivatives for proving and signing to complete transaction creation. + orchard_saks: Vec, #[cfg(feature = "zfuture")] tze_builder: TzeBuilder<'a, TransactionData>, #[cfg(not(feature = "zfuture"))] @@ -190,8 +212,12 @@ impl<'a, P: consensus::Parameters> Builder<'a, P, OsRng> { /// /// The expiry height will be set to the given height plus the default transaction /// expiry delta (20 blocks). - pub fn new(params: P, target_height: BlockHeight) -> Self { - Builder::new_with_rng(params, target_height, OsRng) + pub fn new( + params: P, + target_height: BlockHeight, + orchard_anchor: Option, + ) -> Self { + Builder::new_with_rng(params, target_height, orchard_anchor, OsRng) } } @@ -202,18 +228,36 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { /// # Default values /// /// The expiry height will be set to the given height plus the default transaction - /// expiry delta. - pub fn new_with_rng(params: P, target_height: BlockHeight, rng: R) -> Builder<'a, P, R> { - Self::new_internal(params, rng, target_height) + /// expiry delta (20 blocks). + pub fn new_with_rng( + params: P, + target_height: BlockHeight, + orchard_anchor: Option, + rng: R, + ) -> Builder<'a, P, R> { + let orchard_builder = if params.is_nu_active(NetworkUpgrade::Nu5, target_height) { + orchard_anchor.map(|anchor| { + orchard::builder::Builder::new( + orchard::bundle::Flags::from_parts(true, true), + anchor, + ) + }) + } else { + None + }; + + Self::new_internal(params, rng, target_height, orchard_builder) } } -impl<'a, P: consensus::Parameters, R: RngCore> Builder<'a, P, R> { +impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { /// Common utility function for builder construction. - /// - /// WARNING: THIS MUST REMAIN PRIVATE AS IT ALLOWS CONSTRUCTION - /// OF BUILDERS WITH NON-CryptoRng RNGs - fn new_internal(params: P, rng: R, target_height: BlockHeight) -> Builder<'a, P, R> { + fn new_internal( + params: P, + rng: R, + target_height: BlockHeight, + orchard_builder: Option, + ) -> Self { Builder { params: params.clone(), rng, @@ -221,6 +265,8 @@ impl<'a, P: consensus::Parameters, R: RngCore> Builder<'a, P, R> { expiry_height: target_height + DEFAULT_TX_EXPIRY_DELTA, transparent_builder: TransparentBuilder::empty(), sapling_builder: SaplingBuilder::new(params, target_height), + orchard_builder, + orchard_saks: Vec::new(), #[cfg(feature = "zfuture")] tze_builder: TzeBuilder::empty(), #[cfg(not(feature = "zfuture"))] @@ -229,6 +275,48 @@ impl<'a, P: consensus::Parameters, R: RngCore> Builder<'a, P, R> { } } + /// Adds an Orchard note to be spent in this bundle. + /// + /// Returns an error if the given Merkle path does not have the required anchor for + /// the given note. + pub fn add_orchard_spend( + &mut self, + sk: orchard::keys::SpendingKey, + note: orchard::Note, + merkle_path: orchard::tree::MerklePath, + ) -> Result<(), Error> { + self.orchard_builder + .as_mut() + .ok_or(Error::OrchardAnchorNotAvailable)? + .add_spend(orchard::keys::FullViewingKey::from(&sk), note, merkle_path) + .map_err(Error::OrchardSpend)?; + + self.orchard_saks + .push(orchard::keys::SpendAuthorizingKey::from(&sk)); + + Ok(()) + } + + /// Adds an Orchard recipient to the transaction. + pub fn add_orchard_output( + &mut self, + ovk: Option, + recipient: orchard::Address, + value: u64, + memo: MemoBytes, + ) -> Result<(), Error> { + self.orchard_builder + .as_mut() + .ok_or(Error::OrchardAnchorNotAvailable)? + .add_recipient( + ovk, + recipient, + orchard::value::NoteValue::from_raw(value), + Some(*memo.as_array()), + ) + .map_err(Error::OrchardRecipient) + } + /// Adds a Sapling note to be spent in this transaction. /// /// Returns an error if the given Merkle path does not have the same anchor as the @@ -295,11 +383,18 @@ impl<'a, P: consensus::Parameters, R: RngCore> Builder<'a, P, R> { self.progress_notifier = Some(progress_notifier); } - /// Returns the sum of the transparent, Sapling, and TZE value balances. + /// Returns the sum of the transparent, Sapling, Orchard, and TZE value balances. fn value_balance(&self) -> Result { let value_balances = [ self.transparent_builder.value_balance()?, self.sapling_builder.value_balance(), + if let Some(builder) = &self.orchard_builder { + builder + .value_balance() + .map_err(|_| BalanceError::Overflow)? + } else { + Amount::zero() + }, #[cfg(feature = "zfuture")] self.tze_builder.value_balance()?, ]; @@ -327,6 +422,17 @@ impl<'a, P: consensus::Parameters, R: RngCore> Builder<'a, P, R> { self.transparent_builder.outputs(), self.sapling_builder.inputs().len(), self.sapling_builder.bundle_output_count(), + match std::cmp::max( + self.orchard_builder + .as_ref() + .map_or_else(|| 0, |builder| builder.outputs().len()), + self.orchard_builder + .as_ref() + .map_or_else(|| 0, |builder| builder.spends().len()), + ) { + 1 => 2, + n => n, + }, ) .map_err(Error::Fee)?; self.build_internal(prover, fee) @@ -400,6 +506,13 @@ impl<'a, P: consensus::Parameters, R: RngCore> Builder<'a, P, R> { ) .map_err(Error::SaplingBuild)?; + let orchard_bundle: Option> = + if let Some(builder) = self.orchard_builder { + Some(builder.build(&mut rng).map_err(Error::OrchardBuild)?) + } else { + None + }; + #[cfg(feature = "zfuture")] let (tze_bundle, tze_signers) = self.tze_builder.build(); @@ -411,7 +524,7 @@ impl<'a, P: consensus::Parameters, R: RngCore> Builder<'a, P, R> { transparent_bundle, sprout_bundle: None, sapling_bundle, - orchard_bundle: None, + orchard_bundle, #[cfg(feature = "zfuture")] tze_bundle, }; @@ -456,6 +569,21 @@ impl<'a, P: consensus::Parameters, R: RngCore> Builder<'a, P, R> { None => (None, SaplingMetadata::empty()), }; + let orchard_bundle = unauthed_tx + .orchard_bundle + .map(|b| { + b.create_proof(&orchard::circuit::ProvingKey::build(), &mut rng) + .and_then(|b| { + b.apply_signatures( + &mut rng, + *shielded_sig_commitment.as_ref(), + &self.orchard_saks, + ) + }) + }) + .transpose() + .map_err(Error::OrchardBuild)?; + let authorized_tx = TransactionData { version: unauthed_tx.version, consensus_branch_id: unauthed_tx.consensus_branch_id, @@ -464,7 +592,7 @@ impl<'a, P: consensus::Parameters, R: RngCore> Builder<'a, P, R> { transparent_bundle, sprout_bundle: unauthed_tx.sprout_bundle, sapling_bundle, - orchard_bundle: None, + orchard_bundle, #[cfg(feature = "zfuture")] tze_bundle, }; @@ -511,13 +639,15 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> ExtensionTxBuilder<'a #[cfg(any(test, feature = "test-dependencies"))] mod testing { use rand::RngCore; + use rand_core::CryptoRng; use std::convert::Infallible; use super::{Builder, Error, SaplingMetadata}; use crate::{ consensus::{self, BlockHeight}, sapling::prover::mock::MockTxProver, - transaction::{fees::fixed, Transaction}, + transaction::fees::fixed, + transaction::Transaction, }; impl<'a, P: consensus::Parameters, R: RngCore> Builder<'a, P, R> { @@ -530,10 +660,34 @@ mod testing { /// expiry delta. /// /// WARNING: DO NOT USE IN PRODUCTION - pub fn test_only_new_with_rng(params: P, height: BlockHeight, rng: R) -> Builder<'a, P, R> { - Self::new_internal(params, rng, height) + pub fn test_only_new_with_rng( + params: P, + height: BlockHeight, + rng: R, + ) -> Builder<'a, P, impl RngCore + CryptoRng> { + struct FakeCryptoRng(R); + impl CryptoRng for FakeCryptoRng {} + impl RngCore for FakeCryptoRng { + fn next_u32(&mut self) -> u32 { + self.0.next_u32() + } + + fn next_u64(&mut self) -> u64 { + self.0.next_u64() + } + + fn fill_bytes(&mut self, dest: &mut [u8]) { + self.0.fill_bytes(dest) + } + + fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), rand_core::Error> { + self.0.try_fill_bytes(dest) + } + } + Builder::new_internal(params, FakeCryptoRng(rng), height, None) } - + } + impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { pub fn mock_build(self) -> Result<(Transaction, SaplingMetadata), Error> { #[allow(deprecated)] self.build(&MockTxProver, &fixed::FeeRule::standard()) @@ -588,7 +742,7 @@ mod tests { .activation_height(NetworkUpgrade::Sapling) .unwrap(); - let mut builder = Builder::new(TEST_NETWORK, sapling_activation_height); + let mut builder = Builder::new(TEST_NETWORK, sapling_activation_height, None); assert_eq!( builder.add_sapling_output( Some(ovk), @@ -625,6 +779,8 @@ mod tests { #[cfg(not(feature = "zfuture"))] tze_builder: std::marker::PhantomData, progress_notifier: None, + orchard_builder: None, + orchard_saks: Vec::new(), }; let tsk = AccountPrivKey::from_seed(&TEST_NETWORK, &[0u8; 32], AccountId::from(0)).unwrap(); @@ -676,7 +832,7 @@ mod tests { let tx_height = TEST_NETWORK .activation_height(NetworkUpgrade::Sapling) .unwrap(); - let mut builder = Builder::new(TEST_NETWORK, tx_height); + let mut builder = Builder::new(TEST_NETWORK, tx_height, None); // Create a tx with a sapling spend. binding_sig should be present builder @@ -703,7 +859,7 @@ mod tests { let tx_height = TEST_NETWORK .activation_height(NetworkUpgrade::Sapling) .unwrap(); - let mut builder = Builder::new(TEST_NETWORK, tx_height); + let mut builder = Builder::new(TEST_NETWORK, tx_height, None); assert_eq!( builder.add_transparent_output( &TransparentAddress::PublicKey([0; 20]), @@ -728,7 +884,7 @@ mod tests { // Fails with no inputs or outputs // 0.0001 t-ZEC fee { - let builder = Builder::new(TEST_NETWORK, tx_height); + let builder = Builder::new(TEST_NETWORK, tx_height, None); assert_matches!( builder.mock_build(), Err(Error::InsufficientFunds(MINIMUM_FEE)) @@ -742,7 +898,7 @@ mod tests { // Fail if there is only a Sapling output // 0.0005 z-ZEC out, 0.0001 t-ZEC fee { - let mut builder = Builder::new(TEST_NETWORK, tx_height); + let mut builder = Builder::new(TEST_NETWORK, tx_height, None); builder .add_sapling_output( ovk, @@ -761,7 +917,7 @@ mod tests { // Fail if there is only a transparent output // 0.0005 t-ZEC out, 0.0001 t-ZEC fee { - let mut builder = Builder::new(TEST_NETWORK, tx_height); + let mut builder = Builder::new(TEST_NETWORK, tx_height, None); builder .add_transparent_output( &TransparentAddress::PublicKey([0; 20]), @@ -784,7 +940,7 @@ mod tests { // Fail if there is insufficient input // 0.0003 z-ZEC out, 0.0002 t-ZEC out, 0.0001 t-ZEC fee, 0.00059999 z-ZEC in { - let mut builder = Builder::new(TEST_NETWORK, tx_height); + let mut builder = Builder::new(TEST_NETWORK, tx_height, None); builder .add_sapling_spend( extsk.clone(), @@ -825,7 +981,7 @@ mod tests { // (Still fails because we are using a MockTxProver which doesn't correctly // compute bindingSig.) { - let mut builder = Builder::new(TEST_NETWORK, tx_height); + let mut builder = Builder::new(TEST_NETWORK, tx_height, None); builder .add_sapling_spend( extsk.clone(), diff --git a/zcash_primitives/src/transaction/components/orchard.rs b/zcash_primitives/src/transaction/components/orchard.rs index 440e8ab8b4..9f7df66b6a 100644 --- a/zcash_primitives/src/transaction/components/orchard.rs +++ b/zcash_primitives/src/transaction/components/orchard.rs @@ -20,14 +20,6 @@ pub const FLAG_SPENDS_ENABLED: u8 = 0b0000_0001; pub const FLAG_OUTPUTS_ENABLED: u8 = 0b0000_0010; pub const FLAGS_EXPECTED_UNSET: u8 = !(FLAG_SPENDS_ENABLED | FLAG_OUTPUTS_ENABLED); -/// Marker for a bundle with no proofs or signatures. -#[derive(Debug)] -pub struct Unauthorized; - -impl Authorization for Unauthorized { - type SpendAuth = (); -} - pub trait MapAuth { fn map_spend_auth(&self, s: A::SpendAuth) -> B::SpendAuth; fn map_authorization(&self, a: A) -> B; diff --git a/zcash_primitives/src/transaction/fees.rs b/zcash_primitives/src/transaction/fees.rs index e29913acc1..515c74f272 100644 --- a/zcash_primitives/src/transaction/fees.rs +++ b/zcash_primitives/src/transaction/fees.rs @@ -21,6 +21,7 @@ pub trait FeeRule { /// Implementations of this method should compute the fee amount given exactly the inputs and /// outputs specified, and should NOT compute speculative fees given any additional change /// outputs that may need to be created in order for inputs and outputs to balance. + #[allow(clippy::too_many_arguments)] fn fee_required( &self, params: &P, @@ -29,6 +30,7 @@ pub trait FeeRule { transparent_outputs: &[impl transparent::OutputView], sapling_input_count: usize, sapling_output_count: usize, + orchard_action_count: usize, ) -> Result; } diff --git a/zcash_primitives/src/transaction/fees/fixed.rs b/zcash_primitives/src/transaction/fees/fixed.rs index c3028dcace..0b2278a544 100644 --- a/zcash_primitives/src/transaction/fees/fixed.rs +++ b/zcash_primitives/src/transaction/fees/fixed.rs @@ -56,6 +56,7 @@ impl super::FeeRule for FeeRule { _transparent_outputs: &[impl transparent::OutputView], _sapling_input_count: usize, _sapling_output_count: usize, + _orchard_action_count: usize, ) -> Result { Ok(self.fixed_fee) } diff --git a/zcash_primitives/src/transaction/fees/zip317.rs b/zcash_primitives/src/transaction/fees/zip317.rs index 60fcd767f9..21ecf4cb95 100644 --- a/zcash_primitives/src/transaction/fees/zip317.rs +++ b/zcash_primitives/src/transaction/fees/zip317.rs @@ -129,6 +129,7 @@ impl super::FeeRule for FeeRule { transparent_outputs: &[impl transparent::OutputView], sapling_input_count: usize, sapling_output_count: usize, + orchard_action_count: usize, ) -> Result { let non_p2pkh_inputs: Vec<_> = transparent_inputs .iter() @@ -151,7 +152,8 @@ impl super::FeeRule for FeeRule { let logical_actions = max( ceildiv(t_in_total_size, self.p2pkh_standard_input_size), ceildiv(t_out_total_size, self.p2pkh_standard_output_size), - ) + max(sapling_input_count, sapling_output_count); + ) + max(sapling_input_count, sapling_output_count) + + orchard_action_count; (self.marginal_fee * max(self.grace_actions, logical_actions)) .ok_or_else(|| BalanceError::Overflow.into()) diff --git a/zcash_primitives/src/transaction/mod.rs b/zcash_primitives/src/transaction/mod.rs index a500b54bb7..5d3ebf365e 100644 --- a/zcash_primitives/src/transaction/mod.rs +++ b/zcash_primitives/src/transaction/mod.rs @@ -270,7 +270,8 @@ pub struct Unauthorized; impl Authorization for Unauthorized { type TransparentAuth = transparent::builder::Unauthorized; type SaplingAuth = sapling::builder::Unauthorized; - type OrchardAuth = orchard_serialization::Unauthorized; + type OrchardAuth = + orchard::builder::InProgress; #[cfg(feature = "zfuture")] type TzeAuth = tze::builder::Unauthorized; From 95abfe58367eb4d4ce49bb01c23a6b1577c6a5a6 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 23 Jun 2023 14:56:49 -0600 Subject: [PATCH 0016/1122] Improve documentation for `zcash_primitives::transaction::builder::Error::OrchardAnchorNotAvailable` Co-authored-by: str4d --- zcash_primitives/src/transaction/builder.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zcash_primitives/src/transaction/builder.rs b/zcash_primitives/src/transaction/builder.rs index 953c1c6aac..b686b7242b 100644 --- a/zcash_primitives/src/transaction/builder.rs +++ b/zcash_primitives/src/transaction/builder.rs @@ -72,8 +72,8 @@ pub enum Error { OrchardSpend(orchard::builder::SpendError), /// An error occurred in adding an Orchard Output a transaction. OrchardRecipient(orchard::builder::OutputError), - /// Orchard is not yet active on the network, and the user attempted to add - /// Orchard parts to a transaction. + /// The builder was constructed either without an Orchard anchor or before NU5 + /// activation, but an Orchard spend or recipient was added. OrchardAnchorNotAvailable, /// An error occurred in constructing the TZE parts of a transaction. #[cfg(feature = "zfuture")] From 1b4017e0d1fa349b721a65fed85ee4905e4d6e46 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 23 Jun 2023 15:16:10 -0600 Subject: [PATCH 0017/1122] Apply suggestions from code review Co-authored-by: str4d --- zcash_primitives/CHANGELOG.md | 9 +++++---- zcash_primitives/src/transaction/builder.rs | 16 +++++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 7e1c46ddfa..47c2f3a4b6 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -15,11 +15,12 @@ and this library adheres to Rust's notion of ### Changed - `zcash_primitives::transaction`: - - `builder::Builder::{new, new_with_rng}` now takes an optional `orchard_anchor` + - `builder::Builder::{new, new_with_rng}` now take an optional `orchard_anchor` argument which must be provided in order to enable Orchard spends and recipients. - - `builder::Builder::test_only_new_with_rng` - now returns an existential type: `Builder<'a, P, impl RngCore + CryptoRng>` - instead of `Builder<'a, P, R>` + - All `builder::Builder` methods now require the bound `R: CryptoRng` on + `Builder<'a, P, R>`. A non-`CryptoRng` randomness source is still accepted + by `builder::Builder::test_only_new_with_rng`, which **MUST NOT** be used in + production. - `builder::Error` has several additional variants for Orchard-related errors. - `fees::FeeRule::fee_required` now takes an additional argument, `orchard_action_count` diff --git a/zcash_primitives/src/transaction/builder.rs b/zcash_primitives/src/transaction/builder.rs index b686b7242b..2c4db37f15 100644 --- a/zcash_primitives/src/transaction/builder.rs +++ b/zcash_primitives/src/transaction/builder.rs @@ -68,9 +68,9 @@ pub enum Error { SaplingBuild(sapling_builder::Error), /// An error occurred in constructing the Orchard parts of a transaction. OrchardBuild(orchard::builder::BuildError), - /// An error occurred in adding an Orchard Spend a transaction. + /// An error occurred in adding an Orchard Spend to a transaction. OrchardSpend(orchard::builder::SpendError), - /// An error occurred in adding an Orchard Output a transaction. + /// An error occurred in adding an Orchard Output to a transaction. OrchardRecipient(orchard::builder::OutputError), /// The builder was constructed either without an Orchard anchor or before NU5 /// activation, but an Orchard spend or recipient was added. @@ -98,11 +98,11 @@ impl fmt::Display for Error { Error::TransparentBuild(err) => err.fmt(f), Error::SaplingBuild(err) => err.fmt(f), Error::OrchardBuild(err) => write!(f, "{:?}", err), - Error::OrchardSpend(err) => write!(f, "Could not add orchard spend: {}", err), - Error::OrchardRecipient(err) => write!(f, "Could not add orchard recipient: {}", err), + Error::OrchardSpend(err) => write!(f, "Could not add Orchard spend: {}", err), + Error::OrchardRecipient(err) => write!(f, "Could not add Orchard recipient: {}", err), Error::OrchardAnchorNotAvailable => write!( f, - "Cannot create orchard transactions without an Orchard anchor or before NU5 activation" + "Cannot create Orchard transactions without an Orchard anchor, or before NU5 activation" ), #[cfg(feature = "zfuture")] Error::TzeBuild(err) => err.fmt(f), @@ -248,9 +248,7 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { Self::new_internal(params, rng, target_height, orchard_builder) } -} -impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { /// Common utility function for builder construction. fn new_internal( params: P, @@ -425,10 +423,10 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { match std::cmp::max( self.orchard_builder .as_ref() - .map_or_else(|| 0, |builder| builder.outputs().len()), + .map_or(0, |builder| builder.outputs().len()), self.orchard_builder .as_ref() - .map_or_else(|| 0, |builder| builder.spends().len()), + .map_or(0, |builder| builder.spends().len()), ) { 1 => 2, n => n, From 7fe02f06066be39a16f3010ad98ca214a3d6915a Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 23 Jun 2023 15:24:38 -0600 Subject: [PATCH 0018/1122] Remove needless bound on `FeeRule` from builder `Error` type. --- zcash_primitives/src/transaction/builder.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zcash_primitives/src/transaction/builder.rs b/zcash_primitives/src/transaction/builder.rs index 2c4db37f15..ae016adc05 100644 --- a/zcash_primitives/src/transaction/builder.rs +++ b/zcash_primitives/src/transaction/builder.rs @@ -277,12 +277,12 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { /// /// Returns an error if the given Merkle path does not have the required anchor for /// the given note. - pub fn add_orchard_spend( + pub fn add_orchard_spend( &mut self, sk: orchard::keys::SpendingKey, note: orchard::Note, merkle_path: orchard::tree::MerklePath, - ) -> Result<(), Error> { + ) -> Result<(), Error> { self.orchard_builder .as_mut() .ok_or(Error::OrchardAnchorNotAvailable)? @@ -296,13 +296,13 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { } /// Adds an Orchard recipient to the transaction. - pub fn add_orchard_output( + pub fn add_orchard_output( &mut self, ovk: Option, recipient: orchard::Address, value: u64, memo: MemoBytes, - ) -> Result<(), Error> { + ) -> Result<(), Error> { self.orchard_builder .as_mut() .ok_or(Error::OrchardAnchorNotAvailable)? From 32e2991c4d03bb8de1ce0d161492e157487aa098 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 3 Apr 2023 13:53:43 -0600 Subject: [PATCH 0019/1122] zcash_client_backend: Add note commitment tree sizes to `CompactBlock` serialization. --- zcash_client_backend/proto/compact_formats.proto | 2 ++ zcash_client_backend/src/proto/compact_formats.rs | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/zcash_client_backend/proto/compact_formats.proto b/zcash_client_backend/proto/compact_formats.proto index 077537c606..eac2b2f2f3 100644 --- a/zcash_client_backend/proto/compact_formats.proto +++ b/zcash_client_backend/proto/compact_formats.proto @@ -22,6 +22,8 @@ message CompactBlock { uint32 time = 5; // Unix epoch time when the block was mined bytes header = 6; // (hash, prevHash, and time) OR (full header) repeated CompactTx vtx = 7; // zero or more compact transactions from this block + uint32 saplingCommitmentTreeSize = 8; // the size of the Sapling note commitment tree as of the end of this block + uint32 orchardCommitmentTreeSize = 9; // the size of the Orchard note commitment tree as of the end of this block } // CompactTx contains the minimum information for a wallet to know if this transaction diff --git a/zcash_client_backend/src/proto/compact_formats.rs b/zcash_client_backend/src/proto/compact_formats.rs index 056764b78a..c8d45173c4 100644 --- a/zcash_client_backend/src/proto/compact_formats.rs +++ b/zcash_client_backend/src/proto/compact_formats.rs @@ -26,6 +26,12 @@ pub struct CompactBlock { /// zero or more compact transactions from this block #[prost(message, repeated, tag = "7")] pub vtx: ::prost::alloc::vec::Vec, + /// the size of the Sapling note commitment tree as of the end of this block + #[prost(uint32, tag = "8")] + pub sapling_commitment_tree_size: u32, + /// the size of the Orchard note commitment tree as of the end of this block + #[prost(uint32, tag = "9")] + pub orchard_commitment_tree_size: u32, } /// CompactTx contains the minimum information for a wallet to know if this transaction /// is relevant to it (either pays to it or spends from it) via shielded elements From 3de60c7b91957c7076f85908455286191d170ffb Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 28 Jun 2023 17:25:57 +0100 Subject: [PATCH 0020/1122] Replace `directories` dependency with `home, known-folders, xdg` `directories 5.0.1` added a dependency on `option-ext`, which is licensed as MPL (a copyleft license). The replacement dependencies are all licensed as `MIT OR Apache-2.0`. --- zcash_proofs/CHANGELOG.md | 3 +++ zcash_proofs/Cargo.toml | 5 ++++- zcash_proofs/src/lib.rs | 26 +++++++++++++++++--------- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/zcash_proofs/CHANGELOG.md b/zcash_proofs/CHANGELOG.md index f44adad6b0..8e80daf707 100644 --- a/zcash_proofs/CHANGELOG.md +++ b/zcash_proofs/CHANGELOG.md @@ -6,6 +6,9 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Replaced internal `directories` dependency which now transitively depends on + MPL-licensed code. ## [0.12.0] - 2023-06-06 ### Changed diff --git a/zcash_proofs/Cargo.toml b/zcash_proofs/Cargo.toml index 9930080d1f..2e91476a54 100644 --- a/zcash_proofs/Cargo.toml +++ b/zcash_proofs/Cargo.toml @@ -34,9 +34,11 @@ tracing = "0.1" # Dependencies used internally: # (Breaking upgrades to these are usually backwards-compatible, but check MSRVs.) blake2b_simd = "1" -directories = { version = "5", optional = true } +home = { version = "0.5", optional = true } +known-folders = { version = "1", optional = true } redjubjub = "0.7" wagyu-zcash-parameters = { version = "0.2", optional = true } +xdg = { version = "2.5", optional = true } [dev-dependencies] byteorder = "1" @@ -49,6 +51,7 @@ pprof = { version = "0.11", features = ["criterion", "flamegraph"] } # MSRV 1.56 [features] default = ["local-prover", "multicore"] bundled-prover = ["wagyu-zcash-parameters"] +directories = ["dep:home", "dep:known-folders", "dep:xdg"] download-params = ["minreq", "directories"] local-prover = ["directories"] multicore = ["bellman/multicore", "zcash_primitives/multicore"] diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 346fa48ad3..a666f6e513 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -15,8 +15,6 @@ use std::fs::File; use std::io::{self, BufReader}; use std::path::Path; -#[cfg(feature = "directories")] -use directories::BaseDirs; #[cfg(feature = "directories")] use std::path::PathBuf; @@ -77,13 +75,23 @@ pub struct SaplingParameterPaths { #[cfg(feature = "directories")] #[cfg_attr(docsrs, doc(cfg(feature = "directories")))] pub fn default_params_folder() -> Option { - BaseDirs::new().map(|base_dirs| { - if cfg!(any(windows, target_os = "macos")) { - base_dirs.data_dir().join("ZcashParams") - } else { - base_dirs.home_dir().join(".zcash-params") - } - }) + #[cfg(windows)] + { + use known_folders::{get_known_folder_path, KnownFolder}; + get_known_folder_path(KnownFolder::RoamingAppData).map(|base| base.join("ZcashParams")) + } + + #[cfg(target_os = "macos")] + { + xdg::BaseDirectories::new() + .ok() + .map(|base_dirs| base_dirs.get_data_home().join("ZcashParams")) + } + + #[cfg(not(any(windows, target_os = "macos")))] + { + home::home_dir().map(|base| base.join(".zcash-params")) + } } /// Download the Zcash Sapling parameters if needed, and store them in the default location. From 6f4d68f93580966b5cba575f1f450fffbe3e8b97 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 28 Jun 2023 17:26:47 +0100 Subject: [PATCH 0021/1122] zcash_proofs 0.12.1 --- zcash_proofs/CHANGELOG.md | 2 ++ zcash_proofs/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/zcash_proofs/CHANGELOG.md b/zcash_proofs/CHANGELOG.md index 8e80daf707..339863d34c 100644 --- a/zcash_proofs/CHANGELOG.md +++ b/zcash_proofs/CHANGELOG.md @@ -6,6 +6,8 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [0.12.1] - 2023-06-28 ### Changed - Replaced internal `directories` dependency which now transitively depends on MPL-licensed code. diff --git a/zcash_proofs/Cargo.toml b/zcash_proofs/Cargo.toml index 2e91476a54..e110ba490e 100644 --- a/zcash_proofs/Cargo.toml +++ b/zcash_proofs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_proofs" description = "Zcash zk-SNARK circuits and proving APIs" -version = "0.12.0" +version = "0.12.1" authors = [ "Jack Grigg ", ] From 3e358bc1c95d7b0f4ce8f662ac1668865661aed1 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 3 Apr 2023 13:53:43 -0600 Subject: [PATCH 0022/1122] zcash_client_backend: Use `shardtree` for note commitments in block scanning. Also adds a skeleton `zcash_client_sqlite` implementation of `shardtree::ShardStore` and a skeleton migration for related database changes. --- Cargo.toml | 5 + zcash_client_backend/Cargo.toml | 1 + zcash_client_backend/src/data_api.rs | 155 ++++++++---- zcash_client_backend/src/data_api/chain.rs | 160 ++++-------- .../src/data_api/chain/error.rs | 12 + zcash_client_backend/src/data_api/error.rs | 33 ++- zcash_client_backend/src/data_api/wallet.rs | 146 +++++++---- .../src/data_api/wallet/input_selection.rs | 53 ++-- zcash_client_backend/src/wallet.rs | 16 +- zcash_client_backend/src/welding_rig.rs | 235 ++++++++++-------- zcash_client_sqlite/Cargo.toml | 1 + zcash_client_sqlite/src/chain.rs | 45 +++- zcash_client_sqlite/src/error.rs | 11 + zcash_client_sqlite/src/lib.rs | 234 +++++++++-------- zcash_client_sqlite/src/wallet.rs | 79 ++++-- zcash_client_sqlite/src/wallet/init.rs | 17 +- .../src/wallet/init/migrations.rs | 2 + .../init/migrations/shardtree_support.rs | 56 +++++ zcash_client_sqlite/src/wallet/sapling.rs | 222 +++++++---------- .../src/wallet/sapling/commitment_tree.rs | 123 +++++++++ zcash_primitives/src/merkle_tree.rs | 2 +- 21 files changed, 1017 insertions(+), 591 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs create mode 100644 zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs diff --git a/Cargo.toml b/Cargo.toml index 044d879e93..073970f925 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,3 +17,8 @@ members = [ lto = true panic = 'abort' codegen-units = 1 + +[patch.crates-io] +incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "082109deacf8611ee7917732e19b56158bda96d5" } +shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "082109deacf8611ee7917732e19b56158bda96d5" } +orchard = { git = "https://github.com/zcash/orchard.git", rev = "5da41a6bbb44290e353ee4b38bcafe37ffe79ce8" } diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index eb6eb2c1d9..7d0e463823 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -21,6 +21,7 @@ development = ["zcash_proofs"] [dependencies] incrementalmerkletree = { version = "0.4", features = ["legacy-api"] } +shardtree = "0.0" zcash_address = { version = "0.3", path = "../components/zcash_address" } zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_note_encryption = "0.4" diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 1b3dff2a7c..80ad24f550 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -1,10 +1,12 @@ //! Interfaces for wallet data persistence & low-level wallet utilities. -use std::cmp; use std::collections::HashMap; use std::fmt::Debug; +use std::{cmp, ops::Range}; +use incrementalmerkletree::Retention; use secrecy::SecretVec; +use shardtree::{ShardStore, ShardTree, ShardTreeError}; use zcash_primitives::{ block::BlockHash, consensus::BlockHeight, @@ -29,6 +31,8 @@ pub mod chain; pub mod error; pub mod wallet; +pub const SAPLING_SHARD_HEIGHT: u8 = sapling::NOTE_COMMITMENT_TREE_DEPTH / 2; + pub enum NullifierQuery { Unspent, All, @@ -61,6 +65,30 @@ pub trait WalletRead { /// This will return `Ok(None)` if no block data is present in the database. fn block_height_extrema(&self) -> Result, Self::Error>; + /// Returns the height to which the wallet has been fully scanned. + /// + /// This is the height for which the wallet has fully trial-decrypted this and all preceding + /// blocks above the wallet's birthday height. Along with this height, this method returns + /// metadata describing the state of the wallet's note commitment trees as of the end of that + /// block. + fn fully_scanned_height( + &self, + ) -> Result, Self::Error>; + + /// Returns a vector of suggested scan ranges based upon the current wallet state. + /// + /// This method should only be used in cases where the [`CompactBlock`] data that will be made + /// available to `scan_cached_blocks` for the requested block ranges includes note commitment + /// tree size information for each block; or else the scan is likely to fail if notes belonging + /// to the wallet are detected. + /// + /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock + fn suggest_scan_ranges( + &self, + batch_size: usize, + limit: usize, + ) -> Result>, Self::Error>; + /// Returns the default target height (for the block in which a new /// transaction would be mined) and anchor height (to use for a new /// transaction), given the range of block heights that the backend @@ -165,19 +193,6 @@ pub trait WalletRead { /// Returns a transaction. fn get_transaction(&self, id_tx: Self::TxRef) -> Result; - /// Returns the note commitment tree at the specified block height. - fn get_commitment_tree( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error>; - - /// Returns the incremental witnesses as of the specified block height. - #[allow(clippy::type_complexity)] - fn get_witnesses( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error>; - /// Returns the nullifiers for notes that the wallet is tracking, along with their associated /// account IDs, that are either unspent or have not yet been confirmed as spent (in that a /// spending transaction known to the wallet has not yet been included in a block). @@ -236,12 +251,13 @@ pub trait WalletRead { /// decrypted and extracted from a [`CompactBlock`]. /// /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock -pub struct PrunedBlock<'a> { +pub struct PrunedBlock { pub block_height: BlockHeight, pub block_hash: BlockHash, pub block_time: u32, - pub commitment_tree: &'a sapling::CommitmentTree, - pub transactions: &'a Vec>, + pub transactions: Vec>, + pub sapling_commitment_tree_size: Option, + pub sapling_commitments: Vec<(sapling::Node, Retention)>, } /// A transaction that was detected during scanning of the blockchain, @@ -381,16 +397,14 @@ pub trait WalletWrite: WalletRead { account: AccountId, ) -> Result, Self::Error>; - /// Updates the state of the wallet database by persisting the provided - /// block information, along with the updated witness data that was - /// produced when scanning the block for transactions pertaining to - /// this wallet. + /// Updates the state of the wallet database by persisting the provided block information, + /// along with the note commitments that were detected when scanning the block for transactions + /// pertaining to this wallet. #[allow(clippy::type_complexity)] fn advance_by_block( &mut self, - block: &PrunedBlock, - updated_witnesses: &[(Self::NoteRef, sapling::IncrementalWitness)], - ) -> Result, Self::Error>; + block: PrunedBlock, + ) -> Result, Self::Error>; /// Caches a decrypted transaction in the persistent wallet store. fn store_decrypted_tx( @@ -424,10 +438,31 @@ pub trait WalletWrite: WalletRead { ) -> Result; } +pub trait WalletCommitmentTrees { + type Error; + type SaplingShardStore<'a>: ShardStore< + H = sapling::Node, + CheckpointId = BlockHeight, + Error = Self::Error, + >; + + fn with_sapling_tree_mut(&mut self, callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::SaplingShardStore<'a>, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, + ) -> Result, + E: From>; +} + #[cfg(feature = "test-dependencies")] pub mod testing { use secrecy::{ExposeSecret, SecretVec}; - use std::collections::HashMap; + use shardtree::{MemoryShardStore, ShardTree, ShardTreeError}; + use std::{collections::HashMap, convert::Infallible, ops::Range}; use zcash_primitives::{ block::BlockHash, @@ -449,11 +484,26 @@ pub mod testing { }; use super::{ - DecryptedTransaction, NullifierQuery, PrunedBlock, SentTransaction, WalletRead, WalletWrite, + chain, DecryptedTransaction, NullifierQuery, PrunedBlock, SentTransaction, + WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }; pub struct MockWalletDb { pub network: Network, + pub sapling_tree: ShardTree< + MemoryShardStore, + { SAPLING_SHARD_HEIGHT * 2 }, + SAPLING_SHARD_HEIGHT, + >, + } + + impl MockWalletDb { + pub fn new(network: Network) -> Self { + Self { + network, + sapling_tree: ShardTree::new(MemoryShardStore::empty(), 100), + } + } } impl WalletRead for MockWalletDb { @@ -465,6 +515,20 @@ pub mod testing { Ok(None) } + fn fully_scanned_height( + &self, + ) -> Result, Self::Error> { + Ok(None) + } + + fn suggest_scan_ranges( + &self, + _batch_size: usize, + _limit: usize, + ) -> Result>, Self::Error> { + Ok(vec![]) + } + fn get_min_unspent_height(&self) -> Result, Self::Error> { Ok(None) } @@ -524,21 +588,6 @@ pub mod testing { Err(()) } - fn get_commitment_tree( - &self, - _block_height: BlockHeight, - ) -> Result, Self::Error> { - Ok(None) - } - - #[allow(clippy::type_complexity)] - fn get_witnesses( - &self, - _block_height: BlockHeight, - ) -> Result, Self::Error> { - Ok(Vec::new()) - } - fn get_sapling_nullifiers( &self, _query: NullifierQuery, @@ -613,9 +662,8 @@ pub mod testing { #[allow(clippy::type_complexity)] fn advance_by_block( &mut self, - _block: &PrunedBlock, - _updated_witnesses: &[(Self::NoteRef, sapling::IncrementalWitness)], - ) -> Result, Self::Error> { + _block: PrunedBlock, + ) -> Result, Self::Error> { Ok(vec![]) } @@ -645,4 +693,23 @@ pub mod testing { Ok(0) } } + + impl WalletCommitmentTrees for MockWalletDb { + type Error = Infallible; + type SaplingShardStore<'a> = MemoryShardStore; + + fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::SaplingShardStore<'a>, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, + ) -> Result, + E: From>, + { + callback(&mut self.sapling_tree) + } + } } diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 44736228df..ce0eb2a81b 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -33,9 +33,7 @@ //! # fn test() -> Result<(), Error<(), Infallible, u32>> { //! let network = Network::TestNetwork; //! let block_source = chain_testing::MockBlockSource; -//! let mut db_data = testing::MockWalletDb { -//! network: Network::TestNetwork -//! }; +//! let mut db_data = testing::MockWalletDb::new(Network::TestNetwork); //! //! // 1) Download new CompactBlocks into block_source. //! @@ -79,7 +77,7 @@ //! // At this point, the cache and scanned data are locally consistent (though not //! // necessarily consistent with the latest chain tip - this would be discovered the //! // next time this codepath is executed after new blocks are received). -//! scan_cached_blocks(&network, &block_source, &mut db_data, None) +//! scan_cached_blocks(&network, &block_source, &mut db_data, None, None) //! # } //! # } //! ``` @@ -89,22 +87,34 @@ use std::convert::Infallible; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight}, - sapling::{self, note_encryption::PreparedIncomingViewingKey, Nullifier}, + sapling::{self, note_encryption::PreparedIncomingViewingKey}, zip32::Scope, }; use crate::{ - data_api::{PrunedBlock, WalletWrite}, + data_api::{NullifierQuery, WalletWrite}, proto::compact_formats::CompactBlock, scan::BatchRunner, - wallet::WalletTx, welding_rig::{add_block_to_runner, scan_block_with_runner}, }; pub mod error; use error::{ChainError, Error}; -use super::NullifierQuery; +pub struct CommitmentTreeMeta { + sapling_tree_size: u64, + //TODO: orchard_tree_size: u64 +} + +impl CommitmentTreeMeta { + pub fn from_parts(sapling_tree_size: u64) -> Self { + Self { sapling_tree_size } + } + + pub fn sapling_tree_size(&self) -> u64 { + self.sapling_tree_size + } +} /// This trait provides sequential access to raw blockchain data via a callback-oriented /// API. @@ -212,6 +222,7 @@ pub fn scan_cached_blocks( params: &ParamsT, block_source: &BlockSourceT, data_db: &mut DbT, + from_height: Option, limit: Option, ) -> Result<(), Error> where @@ -219,12 +230,6 @@ where BlockSourceT: BlockSource, DbT: WalletWrite, { - // Recall where we synced up to previously. - let mut last_height = data_db - .block_height_extrema() - .map_err(Error::Wallet)? - .map(|(_, max)| max); - // Fetch the UnifiedFullViewingKeys we are tracking let ufvks = data_db .get_unified_full_viewing_keys() @@ -236,25 +241,8 @@ where .filter_map(|(account, ufvk)| ufvk.sapling().map(move |k| (account, k))) .collect(); - // Get the most recent CommitmentTree - let mut tree = last_height.map_or_else( - || Ok(sapling::CommitmentTree::empty()), - |h| { - data_db - .get_commitment_tree(h) - .map(|t| t.unwrap_or_else(sapling::CommitmentTree::empty)) - .map_err(Error::Wallet) - }, - )?; - - // Get most recent incremental witnesses for the notes we are tracking - let mut witnesses = last_height.map_or_else( - || Ok(vec![]), - |h| data_db.get_witnesses(h).map_err(Error::Wallet), - )?; - - // Get the nullifiers for the notes we are tracking - let mut nullifiers = data_db + // Get the nullifiers for the unspent notes we are tracking + let mut sapling_nullifiers = data_db .get_sapling_nullifiers(NullifierQuery::Unspent) .map_err(Error::Wallet)?; @@ -271,8 +259,19 @@ where .map(|(tag, ivk)| (tag, PreparedIncomingViewingKey::new(&ivk))), ); + // Start at either the provided height, or where we synced up to previously. + let (last_scanned_height, commitment_tree_meta) = from_height.map_or_else( + || { + data_db.fully_scanned_height().map_or_else( + |e| Err(Error::Wallet(e)), + |next| Ok(next.map_or_else(|| (None, None), |(h, m)| (Some(h), Some(m)))), + ) + }, + |h| Ok((Some(h), None)), + )?; + block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - last_height, + last_scanned_height, limit, |block: CompactBlock| { add_block_to_runner(params, block, &mut batch_runner); @@ -283,90 +282,35 @@ where batch_runner.flush(); block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - last_height, + last_scanned_height, limit, |block: CompactBlock| { - let current_height = block.height(); - - // Scanned blocks MUST be height-sequential. - if let Some(h) = last_height { - if current_height != (h + 1) { - return Err( - ChainError::block_height_discontinuity(h + 1, current_height).into(), - ); - } - } - - let block_hash = BlockHash::from_slice(&block.hash); - let block_time = block.time; - - let txs: Vec> = { - let mut witness_refs: Vec<_> = witnesses.iter_mut().map(|w| &mut w.1).collect(); - - scan_block_with_runner( - params, - block, - &dfvks, - &nullifiers, - &mut tree, - &mut witness_refs[..], - Some(&mut batch_runner), - ) - }; - - // Enforce that all roots match. This is slow, so only include in debug builds. - #[cfg(debug_assertions)] - { - let cur_root = tree.root(); - for row in &witnesses { - if row.1.root() != cur_root { - return Err( - ChainError::invalid_witness_anchor(current_height, row.0).into() - ); - } - } - for tx in &txs { - for output in tx.sapling_outputs.iter() { - if output.witness().root() != cur_root { - return Err(ChainError::invalid_new_witness_anchor( - current_height, - tx.txid, - output.index(), - output.witness().root(), - ) - .into()); - } - } - } - } - - let new_witnesses = data_db - .advance_by_block( - &(PrunedBlock { - block_height: current_height, - block_hash, - block_time, - commitment_tree: &tree, - transactions: &txs, - }), - &witnesses, - ) - .map_err(Error::Wallet)?; - - let spent_nf: Vec<&Nullifier> = txs + let pruned_block = scan_block_with_runner( + params, + block, + &dfvks, + &sapling_nullifiers, + commitment_tree_meta.as_ref(), + Some(&mut batch_runner), + ) + .map_err(Error::Sync)?; + + let spent_nf: Vec<&sapling::Nullifier> = pruned_block + .transactions .iter() .flat_map(|tx| tx.sapling_spends.iter().map(|spend| spend.nf())) .collect(); - nullifiers.retain(|(_, nf)| !spent_nf.contains(&nf)); - nullifiers.extend(txs.iter().flat_map(|tx| { + + sapling_nullifiers.retain(|(_, nf)| !spent_nf.contains(&nf)); + sapling_nullifiers.extend(pruned_block.transactions.iter().flat_map(|tx| { tx.sapling_outputs .iter() .map(|out| (out.account(), *out.nf())) })); - witnesses.extend(new_witnesses); - - last_height = Some(current_height); + data_db + .advance_by_block(pruned_block) + .map_err(Error::Wallet)?; Ok(()) }, diff --git a/zcash_client_backend/src/data_api/chain/error.rs b/zcash_client_backend/src/data_api/chain/error.rs index b35334c6ac..b28380d13f 100644 --- a/zcash_client_backend/src/data_api/chain/error.rs +++ b/zcash_client_backend/src/data_api/chain/error.rs @@ -5,6 +5,8 @@ use std::fmt::{self, Debug, Display}; use zcash_primitives::{consensus::BlockHeight, sapling, transaction::TxId}; +use crate::welding_rig::SyncError; + /// The underlying cause of a [`ChainError`]. #[derive(Copy, Clone, Debug)] pub enum Cause { @@ -142,6 +144,9 @@ pub enum Error { /// commitments that could not be reconciled with the note commitment tree(s) maintained by the /// wallet. Chain(ChainError), + + /// An error occorred in block scanning. + Sync(SyncError), } impl fmt::Display for Error { @@ -164,6 +169,13 @@ impl fmt::Display for Error { write!(f, "{}", err) } + Error::Sync(SyncError::SaplingTreeSizeUnknown(h)) => { + write!( + f, + "Sync failed due to missing Sapling note commitment tree size at height {}", + h + ) + } } } } diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index 0614a612d6..db4ffb9842 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -1,5 +1,6 @@ //! Types for wallet error handling. +use shardtree::ShardTreeError; use std::error; use std::fmt::{self, Debug, Display}; use zcash_primitives::{ @@ -20,10 +21,13 @@ use zcash_primitives::{legacy::TransparentAddress, zip32::DiversifierIndex}; /// Errors that can occur as a consequence of wallet operations. #[derive(Debug)] -pub enum Error { +pub enum Error { /// An error occurred retrieving data from the underlying data source DataSource(DataSourceError), + /// An error in computations involving the note commitment trees. + CommitmentTree(ShardTreeError), + /// An error in note selection NoteSelection(SelectionError), @@ -60,9 +64,10 @@ pub enum Error { ChildIndexOutOfRange(DiversifierIndex), } -impl fmt::Display for Error +impl fmt::Display for Error where DE: fmt::Display, + CE: fmt::Display, SE: fmt::Display, FE: fmt::Display, N: fmt::Display, @@ -76,6 +81,9 @@ where e ) } + Error::CommitmentTree(e) => { + write!(f, "An error occurred in querying or updating a note commitment tree: {}", e) + } Error::NoteSelection(e) => { write!(f, "Note selection encountered the following error: {}", e) } @@ -120,9 +128,10 @@ where } } -impl error::Error for Error +impl error::Error for Error where DE: Debug + Display + error::Error + 'static, + CE: Debug + Display + error::Error + 'static, SE: Debug + Display + error::Error + 'static, FE: Debug + Display + 'static, N: Debug + Display, @@ -130,6 +139,7 @@ where fn source(&self) -> Option<&(dyn error::Error + 'static)> { match &self { Error::DataSource(e) => Some(e), + Error::CommitmentTree(e) => Some(e), Error::NoteSelection(e) => Some(e), Error::Builder(e) => Some(e), _ => None, @@ -137,19 +147,19 @@ where } } -impl From> for Error { +impl From> for Error { fn from(e: builder::Error) -> Self { Error::Builder(e) } } -impl From for Error { +impl From for Error { fn from(e: BalanceError) -> Self { Error::BalanceError(e) } } -impl From> for Error { +impl From> for Error { fn from(e: InputSelectorError) -> Self { match e { InputSelectorError::DataSource(e) => Error::DataSource(e), @@ -161,18 +171,25 @@ impl From> for Error { available, required, }, + InputSelectorError::SyncRequired => Error::ScanRequired, } } } -impl From for Error { +impl From for Error { fn from(e: sapling::builder::Error) -> Self { Error::Builder(builder::Error::SaplingBuild(e)) } } -impl From for Error { +impl From for Error { fn from(e: transparent::builder::Error) -> Self { Error::Builder(builder::Error::TransparentBuild(e)) } } + +impl From> for Error { + fn from(e: ShardTreeError) -> Self { + Error::CommitmentTree(e) + } +} diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 160529dc6a..b0930d9665 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1,8 +1,9 @@ use std::convert::Infallible; use std::fmt::Debug; +use shardtree::{ShardStore, ShardTree, ShardTreeError}; use zcash_primitives::{ - consensus::{self, NetworkUpgrade}, + consensus::{self, BlockHeight, NetworkUpgrade}, memo::MemoBytes, sapling::{ self, @@ -23,7 +24,8 @@ use crate::{ address::RecipientAddress, data_api::{ error::Error, wallet::input_selection::Proposal, DecryptedTransaction, PoolType, Recipient, - SentTransaction, SentTransactionOutput, WalletWrite, + SentTransaction, SentTransactionOutput, WalletCommitmentTrees, WalletRead, WalletWrite, + SAPLING_SHARD_HEIGHT, }, decrypt_transaction, fees::{self, ChangeValue, DustOutputPolicy}, @@ -122,7 +124,7 @@ where /// # Examples /// /// ``` -/// # #[cfg(feature = "test-dependencies")] +/// # #[cfg(all(feature = "test-dependencies", feature = "local-prover"))] /// # { /// use tempfile::NamedTempFile; /// use zcash_primitives::{ @@ -200,7 +202,8 @@ pub fn create_spend_to_address( ) -> Result< DbT::TxRef, Error< - DbT::Error, + ::Error, + ::Error, GreedyInputSelectorError, Infallible, DbT::NoteRef, @@ -208,7 +211,7 @@ pub fn create_spend_to_address( > where ParamsT: consensus::Parameters + Clone, - DbT: WalletWrite, + DbT: WalletWrite + WalletCommitmentTrees, DbT::NoteRef: Copy + Eq + Ord, { let req = zip321::TransactionRequest::new(vec![Payment { @@ -300,10 +303,16 @@ pub fn spend( min_confirmations: u32, ) -> Result< DbT::TxRef, - Error::Error, DbT::NoteRef>, + Error< + ::Error, + ::Error, + InputsT::Error, + ::Error, + DbT::NoteRef, + >, > where - DbT: WalletWrite, + DbT: WalletWrite + WalletCommitmentTrees, DbT::TxRef: Copy + Debug, DbT::NoteRef: Copy + Eq + Ord, ParamsT: consensus::Parameters + Clone, @@ -323,7 +332,16 @@ where min_confirmations, )?; - create_proposed_transaction(wallet_db, params, prover, usk, ovk_policy, proposal, None) + create_proposed_transaction( + wallet_db, + params, + prover, + usk, + ovk_policy, + proposal, + min_confirmations, + None, + ) } /// Select transaction inputs, compute fees, and construct a proposal for a transaction @@ -331,7 +349,7 @@ where /// [`create_proposed_transaction`]. #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] -pub fn propose_transfer( +pub fn propose_transfer( wallet_db: &mut DbT, params: &ParamsT, spend_from_account: AccountId, @@ -340,7 +358,13 @@ pub fn propose_transfer( min_confirmations: u32, ) -> Result< Proposal, - Error::Error, DbT::NoteRef>, + Error< + DbT::Error, + CommitmentTreeErrT, + InputsT::Error, + ::Error, + DbT::NoteRef, + >, > where DbT: WalletWrite, @@ -348,20 +372,13 @@ where ParamsT: consensus::Parameters + Clone, InputsT: InputSelector, { - // Target the next block, assuming we are up-to-date. - let (target_height, anchor_height) = wallet_db - .get_target_and_anchor_heights(min_confirmations) - .map_err(Error::DataSource) - .and_then(|x| x.ok_or(Error::ScanRequired))?; - input_selector .propose_transaction( params, wallet_db, spend_from_account, - anchor_height, - target_height, request, + min_confirmations, ) .map_err(Error::from) } @@ -369,7 +386,7 @@ where #[cfg(feature = "transparent-inputs")] #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] -pub fn propose_shielding( +pub fn propose_shielding( wallet_db: &mut DbT, params: &ParamsT, input_selector: &InputsT, @@ -378,7 +395,13 @@ pub fn propose_shielding( min_confirmations: u32, ) -> Result< Proposal, - Error::Error, DbT::NoteRef>, + Error< + DbT::Error, + CommitmentTreeErrT, + InputsT::Error, + ::Error, + DbT::NoteRef, + >, > where ParamsT: consensus::Parameters, @@ -386,19 +409,13 @@ where DbT::NoteRef: Copy + Eq + Ord, InputsT: InputSelector, { - let (target_height, latest_anchor) = wallet_db - .get_target_and_anchor_heights(min_confirmations) - .map_err(Error::DataSource) - .and_then(|x| x.ok_or(Error::ScanRequired))?; - input_selector .propose_shielding( params, wallet_db, shielding_threshold, from_addrs, - latest_anchor, - target_height, + min_confirmations, ) .map_err(Error::from) } @@ -417,10 +434,20 @@ pub fn create_proposed_transaction( usk: &UnifiedSpendingKey, ovk_policy: OvkPolicy, proposal: Proposal, + min_confirmations: u32, change_memo: Option, -) -> Result> +) -> Result< + DbT::TxRef, + Error< + ::Error, + ::Error, + InputsErrT, + FeeRuleT::Error, + DbT::NoteRef, + >, +> where - DbT: WalletWrite, + DbT: WalletWrite + WalletCommitmentTrees, DbT::TxRef: Copy + Debug, DbT::NoteRef: Copy + Eq + Ord, ParamsT: consensus::Parameters + Clone, @@ -459,14 +486,25 @@ where // Create the transaction. The type of the proposal ensures that there // are no possible transparent inputs, so we ignore those - let mut builder = Builder::new(params.clone(), proposal.target_height(), None); - - for selected in proposal.sapling_inputs() { - let (note, key, merkle_path) = select_key_for_note(selected, usk.sapling(), &dfvk) + let mut builder = Builder::new(params.clone(), proposal.min_target_height(), None); + + wallet_db.with_sapling_tree_mut::<_, _, Error<_, _, _, _, _>>(|sapling_tree| { + for selected in proposal.sapling_inputs() { + let (note, key, merkle_path) = select_key_for_note( + sapling_tree, + selected, + usk.sapling(), + &dfvk, + min_confirmations + .try_into() + .expect("min_confirmations should never be anywhere close to usize::MAX"), + )? .ok_or(Error::NoteMismatch(selected.note_id))?; - builder.add_sapling_spend(key, selected.diversifier, note, merkle_path)?; - } + builder.add_sapling_spend(key, selected.diversifier, note, merkle_path)?; + } + Ok(()) + })?; #[cfg(feature = "transparent-inputs")] let utxos = { @@ -577,7 +615,7 @@ where tx.sapling_bundle().and_then(|bundle| { try_sapling_note_decryption( params, - proposal.target_height(), + proposal.min_target_height(), &internal_ivk, &bundle.shielded_outputs()[output_index], ) @@ -672,11 +710,17 @@ pub fn shield_transparent_funds( min_confirmations: u32, ) -> Result< DbT::TxRef, - Error::Error, DbT::NoteRef>, + Error< + ::Error, + ::Error, + InputsT::Error, + ::Error, + DbT::NoteRef, + >, > where ParamsT: consensus::Parameters, - DbT: WalletWrite, + DbT: WalletWrite + WalletCommitmentTrees, DbT::NoteRef: Copy + Eq + Ord, InputsT: InputSelector, { @@ -696,17 +740,26 @@ where usk, OvkPolicy::Sender, proposal, + min_confirmations, Some(memo.clone()), ) } -fn select_key_for_note( +#[allow(clippy::type_complexity)] +fn select_key_for_note>( + commitment_tree: &mut ShardTree< + S, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, selected: &ReceivedSaplingNote, extsk: &ExtendedSpendingKey, dfvk: &DiversifiableFullViewingKey, -) -> Option<(sapling::Note, ExtendedSpendingKey, sapling::MerklePath)> { - let merkle_path = selected.witness.path().expect("the tree is not empty"); - + checkpoint_depth: usize, +) -> Result< + Option<(sapling::Note, ExtendedSpendingKey, sapling::MerklePath)>, + ShardTreeError, +> { // Attempt to reconstruct the note being spent using both the internal and external dfvks // corresponding to the unified spending key, checking against the witness we are using // to spend the note that we've used the correct key. @@ -717,13 +770,16 @@ fn select_key_for_note( .diversified_change_address(selected.diversifier) .map(|addr| addr.create_note(selected.note_value.into(), selected.rseed)); - let expected_root = selected.witness.root(); - external_note + let expected_root = commitment_tree.root_at_checkpoint(checkpoint_depth)?; + let merkle_path = commitment_tree + .witness_caching(selected.note_commitment_tree_position, checkpoint_depth)?; + + Ok(external_note .filter(|n| expected_root == merkle_path.root(Node::from_cmu(&n.cmu()))) .map(|n| (n, extsk.clone(), merkle_path.clone())) .or_else(|| { internal_note .filter(|n| expected_root == merkle_path.root(Node::from_cmu(&n.cmu()))) .map(|n| (n, extsk.derive_internal(), merkle_path)) - }) + })) } diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 798b834501..403497c0d5 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -35,6 +35,9 @@ pub enum InputSelectorError { /// Insufficient funds were available to satisfy the payment request that inputs were being /// selected to attempt to satisfy. InsufficientFunds { available: Amount, required: Amount }, + /// The data source does not have enough information to choose an expiry height + /// for the transaction. + SyncRequired, } impl fmt::Display for InputSelectorError { @@ -59,6 +62,7 @@ impl fmt::Display for InputSelectorError write!(f, "No chain data is available."), } } } @@ -71,7 +75,8 @@ pub struct Proposal { sapling_inputs: Vec>, balance: TransactionBalance, fee_rule: FeeRuleT, - target_height: BlockHeight, + min_target_height: BlockHeight, + min_anchor_height: BlockHeight, is_shielding: bool, } @@ -97,8 +102,19 @@ impl Proposal { &self.fee_rule } /// Returns the target height for which the proposal was prepared. - pub fn target_height(&self) -> BlockHeight { - self.target_height + /// + /// The chain must contain at least this many blocks in order for the proposal to + /// be executed. + pub fn min_target_height(&self) -> BlockHeight { + self.min_target_height + } + /// Returns the anchor height used in preparing the proposal. + /// + /// If, at the time that the proposal is executed, the anchor height required to satisfy + /// the minimum confirmation depth is less than this height, the proposal execution + /// API should return an error. + pub fn min_anchor_height(&self) -> BlockHeight { + self.min_anchor_height } /// Returns a flag indicating whether or not the proposed transaction /// is exclusively wallet-internal (if it does not involve any external @@ -146,9 +162,8 @@ pub trait InputSelector { params: &ParamsT, wallet_db: &Self::DataSource, account: AccountId, - anchor_height: BlockHeight, - target_height: BlockHeight, transaction_request: TransactionRequest, + min_confirmations: u32, ) -> Result< Proposal::DataSource as WalletRead>::NoteRef>, InputSelectorError<<::DataSource as WalletRead>::Error, Self::Error>, @@ -172,8 +187,7 @@ pub trait InputSelector { wallet_db: &Self::DataSource, shielding_threshold: NonNegativeAmount, source_addrs: &[TransparentAddress], - confirmed_height: BlockHeight, - target_height: BlockHeight, + min_confirmations: u32, ) -> Result< Proposal::DataSource as WalletRead>::NoteRef>, InputSelectorError<<::DataSource as WalletRead>::Error, Self::Error>, @@ -292,13 +306,18 @@ where params: &ParamsT, wallet_db: &Self::DataSource, account: AccountId, - anchor_height: BlockHeight, - target_height: BlockHeight, transaction_request: TransactionRequest, + min_confirmations: u32, ) -> Result, InputSelectorError> where ParamsT: consensus::Parameters, { + // Target the next block, assuming we are up-to-date. + let (target_height, anchor_height) = wallet_db + .get_target_and_anchor_heights(min_confirmations) + .map_err(InputSelectorError::DataSource) + .and_then(|x| x.ok_or(InputSelectorError::SyncRequired))?; + let mut transparent_outputs = vec![]; let mut sapling_outputs = vec![]; let mut output_total = Amount::zero(); @@ -362,7 +381,8 @@ where sapling_inputs, balance, fee_rule: (*self.change_strategy.fee_rule()).clone(), - target_height, + min_target_height: target_height, + min_anchor_height: anchor_height, is_shielding: false, }); } @@ -405,15 +425,19 @@ where wallet_db: &Self::DataSource, shielding_threshold: NonNegativeAmount, source_addrs: &[TransparentAddress], - confirmed_height: BlockHeight, - target_height: BlockHeight, + min_confirmations: u32, ) -> Result, InputSelectorError> where ParamsT: consensus::Parameters, { + let (target_height, latest_anchor) = wallet_db + .get_target_and_anchor_heights(min_confirmations) + .map_err(InputSelectorError::DataSource) + .and_then(|x| x.ok_or(InputSelectorError::SyncRequired))?; + let mut transparent_inputs: Vec = source_addrs .iter() - .map(|taddr| wallet_db.get_unspent_transparent_outputs(taddr, confirmed_height, &[])) + .map(|taddr| wallet_db.get_unspent_transparent_outputs(taddr, latest_anchor, &[])) .collect::>, _>>() .map_err(InputSelectorError::DataSource)? .into_iter() @@ -458,7 +482,8 @@ where sapling_inputs: vec![], balance, fee_rule: (*self.change_strategy.fee_rule()).clone(), - target_height, + min_target_height: target_height, + min_anchor_height: latest_anchor, is_shielding: true, }) } else { diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index ba58340b3d..c702cfa73a 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -1,6 +1,7 @@ //! Structs representing transaction data scanned from the block chain by a wallet or //! light client. +use incrementalmerkletree::Position; use zcash_note_encryption::EphemeralKeyBytes; use zcash_primitives::{ consensus::BlockHeight, @@ -117,7 +118,7 @@ pub struct WalletSaplingOutput { account: AccountId, note: sapling::Note, is_change: bool, - witness: sapling::IncrementalWitness, + note_commitment_tree_position: Position, nf: N, } @@ -131,7 +132,7 @@ impl WalletSaplingOutput { account: AccountId, note: sapling::Note, is_change: bool, - witness: sapling::IncrementalWitness, + note_commitment_tree_position: Position, nf: N, ) -> Self { Self { @@ -141,7 +142,7 @@ impl WalletSaplingOutput { account, note, is_change, - witness, + note_commitment_tree_position, nf, } } @@ -164,11 +165,8 @@ impl WalletSaplingOutput { pub fn is_change(&self) -> bool { self.is_change } - pub fn witness(&self) -> &sapling::IncrementalWitness { - &self.witness - } - pub fn witness_mut(&mut self) -> &mut sapling::IncrementalWitness { - &mut self.witness + pub fn note_commitment_tree_position(&self) -> Position { + self.note_commitment_tree_position } pub fn nf(&self) -> &N { &self.nf @@ -182,7 +180,7 @@ pub struct ReceivedSaplingNote { pub diversifier: sapling::Diversifier, pub note_value: Amount, pub rseed: sapling::Rseed, - pub witness: sapling::IncrementalWitness, + pub note_commitment_tree_position: Position, } impl sapling_fees::InputView for ReceivedSaplingNote { diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index 1906c133e6..3266d40594 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -1,21 +1,27 @@ //! Tools for scanning a compact representation of the Zcash block chain. +//! +//! TODO: rename this module to `block_scanner` use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; +use incrementalmerkletree::{Position, Retention}; use subtle::{ConditionallySelectable, ConstantTimeEq, CtOption}; use zcash_note_encryption::batch; +use zcash_primitives::consensus::BlockHeight; use zcash_primitives::{ consensus, sapling::{ self, note_encryption::{PreparedIncomingViewingKey, SaplingDomain}, - Node, Note, Nullifier, NullifierDerivingKey, SaplingIvk, + SaplingIvk, }, transaction::components::sapling::CompactOutputDescription, zip32::{sapling::DiversifiableFullViewingKey, AccountId, Scope}, }; +use crate::data_api::chain::CommitmentTreeMeta; +use crate::data_api::PrunedBlock; use crate::{ proto::compact_formats::CompactBlock, scan::{Batch, BatchRunner, Tasks}, @@ -56,16 +62,13 @@ pub trait ScanningKey { /// IVK-based implementations of this trait cannot successfully derive /// nullifiers, in which case `Self::Nf` should be set to the unit type /// and this function is a no-op. - fn sapling_nf( - key: &Self::SaplingNk, - note: &Note, - witness: &sapling::IncrementalWitness, - ) -> Self::Nf; + fn sapling_nf(key: &Self::SaplingNk, note: &sapling::Note, note_position: Position) + -> Self::Nf; } impl ScanningKey for DiversifiableFullViewingKey { type Scope = Scope; - type SaplingNk = NullifierDerivingKey; + type SaplingNk = sapling::NullifierDerivingKey; type SaplingKeys = [(Self::Scope, SaplingIvk, Self::SaplingNk); 2]; type Nf = sapling::Nullifier; @@ -84,16 +87,8 @@ impl ScanningKey for DiversifiableFullViewingKey { ] } - fn sapling_nf( - key: &Self::SaplingNk, - note: &Note, - witness: &sapling::IncrementalWitness, - ) -> Self::Nf { - note.nf( - key, - u64::try_from(witness.position()) - .expect("Sapling note commitment tree position must fit into a u64"), - ) + fn sapling_nf(key: &Self::SaplingNk, note: &sapling::Note, position: Position) -> Self::Nf { + note.nf(key, position.into()) } } @@ -111,7 +106,15 @@ impl ScanningKey for SaplingIvk { [((), self.clone(), ())] } - fn sapling_nf(_key: &Self::SaplingNk, _note: &Note, _witness: &sapling::IncrementalWitness) {} + fn sapling_nf(_key: &Self::SaplingNk, _note: &sapling::Note, _position: Position) {} +} + +/// Errors that can occur in block scanning. +#[derive(Debug)] +pub enum SyncError { + /// The size of the Sapling note commitment tree was not provided as part of a [`CompactBlock`] + /// being scanned, making it impossible to construct the nullifier for a detected note. + SaplingTreeSizeUnknown(BlockHeight), } /// Scans a [`CompactBlock`] with a set of [`ScanningKey`]s. @@ -141,17 +144,15 @@ pub fn scan_block( params: &P, block: CompactBlock, vks: &[(&AccountId, &K)], - nullifiers: &[(AccountId, Nullifier)], - tree: &mut sapling::CommitmentTree, - existing_witnesses: &mut [&mut sapling::IncrementalWitness], -) -> Vec> { + sapling_nullifiers: &[(AccountId, sapling::Nullifier)], + initial_commitment_tree_meta: Option<&CommitmentTreeMeta>, +) -> Result, SyncError> { scan_block_with_runner::<_, _, ()>( params, block, vks, - nullifiers, - tree, - existing_witnesses, + sapling_nullifiers, + initial_commitment_tree_meta, None, ) } @@ -202,21 +203,41 @@ pub(crate) fn scan_block_with_runner< params: &P, block: CompactBlock, vks: &[(&AccountId, &K)], - nullifiers: &[(AccountId, Nullifier)], - tree: &mut sapling::CommitmentTree, - existing_witnesses: &mut [&mut sapling::IncrementalWitness], + nullifiers: &[(AccountId, sapling::Nullifier)], + initial_commitment_tree_meta: Option<&CommitmentTreeMeta>, mut batch_runner: Option<&mut TaggedBatchRunner>, -) -> Vec> { +) -> Result, SyncError> { let mut wtxs: Vec> = vec![]; + let mut sapling_note_commitments: Vec<(sapling::Node, Retention)> = vec![]; let block_height = block.height(); let block_hash = block.hash(); + // It's possible to make progress without a Sapling tree position if we don't have any Sapling + // notes in the block, since we only use the position for constructing nullifiers for our own + // received notes. Thus, we allow it to be optional here, and only produce an error if we try + // to use it. `block.sapling_commitment_tree_size` is expected to be correct as of the end of + // the block, and we can't have a note of ours in a block with no outputs so treating the zero + // default value from the protobuf as `None` is always correct. + let mut sapling_tree_position = if block.sapling_commitment_tree_size == 0 { + initial_commitment_tree_meta.map(|m| (m.sapling_tree_size() + 1).into()) + } else { + let end_position_exclusive = Position::from(u64::from(block.sapling_commitment_tree_size)); + let output_count = block + .vtx + .iter() + .map(|tx| u64::try_from(tx.outputs.len()).unwrap()) + .sum(); + Some(end_position_exclusive - output_count) + }; + for tx in block.vtx.into_iter() { let txid = tx.txid(); let index = tx.index as usize; - // Check for spent notes - // The only step that is not constant-time is the filter() at the end. + // Check for spent notes. The only step that is not constant-time is + // the filter() at the end. + // TODO: However, this is O(|nullifiers| * |notes|); does using + // constant-time operations here really make sense? let shielded_spends: Vec<_> = tx .spends .into_iter() @@ -248,19 +269,8 @@ pub(crate) fn scan_block_with_runner< // Check for incoming notes while incrementing tree and witnesses let mut shielded_outputs: Vec> = vec![]; + let tx_outputs_len = u64::try_from(tx.outputs.len()).unwrap(); { - // Grab mutable references to new witnesses from previous transactions - // in this block so that we can update them. Scoped so we don't hold - // mutable references to wtxs for too long. - let mut block_witnesses: Vec<_> = wtxs - .iter_mut() - .flat_map(|tx| { - tx.sapling_outputs - .iter_mut() - .map(|output| output.witness_mut()) - }) - .collect(); - let decoded = &tx .outputs .into_iter() @@ -292,7 +302,7 @@ pub(crate) fn scan_block_with_runner< "The batch runner and scan_block must use the same set of IVKs.", ); - ((d_note.note, d_note.recipient), a, (*nk).clone()) + (d_note.note, a, (*nk).clone()) }) }) .collect() @@ -312,40 +322,21 @@ pub(crate) fn scan_block_with_runner< .map(PreparedIncomingViewingKey::new) .collect::>(); - batch::try_compact_note_decryption(&ivks, decoded) + batch::try_compact_note_decryption(&ivks, &decoded[..]) .into_iter() .map(|v| { - v.map(|(note_data, ivk_idx)| { + v.map(|((note, _), ivk_idx)| { let (account, _, nk) = &vks[ivk_idx]; - (note_data, *account, (*nk).clone()) + (note, *account, (*nk).clone()) }) }) .collect() }; for (index, ((_, output), dec_output)) in decoded.iter().zip(decrypted).enumerate() { - // Grab mutable references to new witnesses from previous outputs - // in this transaction so that we can update them. Scoped so we - // don't hold mutable references to shielded_outputs for too long. - let new_witnesses: Vec<_> = shielded_outputs - .iter_mut() - .map(|out| out.witness_mut()) - .collect(); - - // Increment tree and witnesses - let node = Node::from_cmu(&output.cmu); - for witness in &mut *existing_witnesses { - witness.append(node).unwrap(); - } - for witness in &mut block_witnesses { - witness.append(node).unwrap(); - } - for witness in new_witnesses { - witness.append(node).unwrap(); - } - tree.append(node).unwrap(); - - if let Some(((note, _), account, nk)) = dec_output { + // Collect block note commitments + let node = sapling::Node::from_cmu(&output.cmu); + if let Some((note, account, nk)) = dec_output { // A note is marked as "change" if the account that received it // also spent notes in the same transaction. This will catch, // for instance: @@ -353,8 +344,10 @@ pub(crate) fn scan_block_with_runner< // - Notes created by consolidation transactions. // - Notes sent from one account to itself. let is_change = spent_from_accounts.contains(&account); - let witness = sapling::IncrementalWitness::from_tree(tree.clone()); - let nf = K::sapling_nf(&nk, ¬e, &witness); + let note_commitment_tree_position = sapling_tree_position + .ok_or(SyncError::SaplingTreeSizeUnknown(block_height))? + + index.try_into().unwrap(); + let nf = K::sapling_nf(&nk, ¬e, note_commitment_tree_position); shielded_outputs.push(WalletSaplingOutput::from_parts( index, @@ -363,9 +356,33 @@ pub(crate) fn scan_block_with_runner< account, note, is_change, - witness, + note_commitment_tree_position, nf, - )) + )); + + sapling_note_commitments.push(( + node, + if index == decoded.len() - 1 { + Retention::Checkpoint { + id: block_height, + is_marked: true, + } + } else { + Retention::Marked + }, + )); + } else { + sapling_note_commitments.push(( + node, + if index == decoded.len() - 1 { + Retention::Checkpoint { + id: block_height, + is_marked: false, + } + } else { + Retention::Ephemeral + }, + )); } } } @@ -378,9 +395,22 @@ pub(crate) fn scan_block_with_runner< sapling_outputs: shielded_outputs, }); } + + sapling_tree_position = sapling_tree_position.map(|pos| pos + tx_outputs_len); } - wtxs + Ok(PrunedBlock { + block_height, + block_hash, + block_time: block.time, + transactions: wtxs, + sapling_commitment_tree_size: if block.sapling_commitment_tree_size == 0 { + None + } else { + Some(block.sapling_commitment_tree_size) + }, + sapling_commitments: sapling_note_commitments, + }) } #[cfg(test)] @@ -396,16 +426,18 @@ mod tests { constants::SPENDING_KEY_GENERATOR, memo::MemoBytes, sapling::{ + self, note_encryption::{sapling_note_encryption, PreparedIncomingViewingKey, SaplingDomain}, util::generate_random_rseed, value::NoteValue, - CommitmentTree, Note, Nullifier, SaplingIvk, + Nullifier, SaplingIvk, }, transaction::components::Amount, zip32::{AccountId, DiversifiableFullViewingKey, ExtendedSpendingKey}, }; use crate::{ + data_api::chain::CommitmentTreeMeta, proto::compact_formats::{ CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }, @@ -455,13 +487,14 @@ mod tests { dfvk: &DiversifiableFullViewingKey, value: Amount, tx_after: bool, + initial_sapling_tree_size: u32, ) -> CompactBlock { let to = dfvk.default_address().1; // Create a fake Note for the account let mut rng = OsRng; let rseed = generate_random_rseed(&Network::TestNetwork, height, &mut rng); - let note = Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); + let note = sapling::Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); let encryptor = sapling_note_encryption::<_, Network>( Some(dfvk.fvk().ovk), note.clone(), @@ -514,6 +547,9 @@ mod tests { cb.vtx.push(tx); } + cb.sapling_commitment_tree_size = initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); + cb } @@ -530,10 +566,10 @@ mod tests { &dfvk, Amount::from_u64(5).unwrap(), false, + 0, ); assert_eq!(cb.vtx.len(), 2); - let mut tree = CommitmentTree::empty(); let mut batch_runner = if scan_multithreaded { let mut runner = BatchRunner::<_, _, _, ()>::new( 10, @@ -551,15 +587,16 @@ mod tests { None }; - let txs = scan_block_with_runner( + let pruned_block = scan_block_with_runner( &Network::TestNetwork, cb, &[(&account, &dfvk)], &[], - &mut tree, - &mut [], + Some(&CommitmentTreeMeta::from_parts(0)), batch_runner.as_mut(), - ); + ) + .unwrap(); + let txs = pruned_block.transactions; assert_eq!(txs.len(), 1); let tx = &txs[0]; @@ -569,9 +606,6 @@ mod tests { assert_eq!(tx.sapling_outputs[0].index(), 0); assert_eq!(tx.sapling_outputs[0].account(), account); assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); - - // Check that the witness root matches - assert_eq!(tx.sapling_outputs[0].witness().root(), tree.root()); } go(false); @@ -591,10 +625,10 @@ mod tests { &dfvk, Amount::from_u64(5).unwrap(), true, + 0, ); assert_eq!(cb.vtx.len(), 3); - let mut tree = CommitmentTree::empty(); let mut batch_runner = if scan_multithreaded { let mut runner = BatchRunner::<_, _, _, ()>::new( 10, @@ -612,15 +646,16 @@ mod tests { None }; - let txs = scan_block_with_runner( + let pruned_block = scan_block_with_runner( &Network::TestNetwork, cb, &[(&AccountId::from(0), &dfvk)], &[], - &mut tree, - &mut [], + Some(&CommitmentTreeMeta::from_parts(0)), batch_runner.as_mut(), - ); + ) + .unwrap(); + let txs = pruned_block.transactions; assert_eq!(txs.len(), 1); let tx = &txs[0]; @@ -630,9 +665,6 @@ mod tests { assert_eq!(tx.sapling_outputs[0].index(), 0); assert_eq!(tx.sapling_outputs[0].account(), AccountId::from(0)); assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); - - // Check that the witness root matches - assert_eq!(tx.sapling_outputs[0].witness().root(), tree.root()); } go(false); @@ -646,19 +678,26 @@ mod tests { let nf = Nullifier([7; 32]); let account = AccountId::from(12); - let cb = fake_compact_block(1u32.into(), nf, &dfvk, Amount::from_u64(5).unwrap(), false); + let cb = fake_compact_block( + 1u32.into(), + nf, + &dfvk, + Amount::from_u64(5).unwrap(), + false, + 0, + ); assert_eq!(cb.vtx.len(), 2); let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; - let mut tree = CommitmentTree::empty(); - let txs = scan_block( + let pruned_block = scan_block( &Network::TestNetwork, cb, &vks[..], &[(account, nf)], - &mut tree, - &mut [], - ); + Some(&CommitmentTreeMeta::from_parts(0)), + ) + .unwrap(); + let txs = pruned_block.transactions; assert_eq!(txs.len(), 1); let tx = &txs[0]; diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index de71727b3e..88585e0d22 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -15,6 +15,7 @@ rust-version = "1.65" [dependencies] incrementalmerkletree = { version = "0.4", features = ["legacy-api"] } +shardtree = { version = "0.0", features = ["legacy-api"] } zcash_client_backend = { version = "0.9", path = "../zcash_client_backend" } zcash_primitives = { version = "0.12", path = "../zcash_primitives", default-features = false } diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 81a0e028a6..11e065f9e4 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -314,6 +314,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), + 0, ); insert_into_cache(&db_cache, &cb); @@ -328,7 +329,7 @@ mod tests { assert_matches!(validate_chain_result, Ok(())); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -340,6 +341,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(7).unwrap(), + 1, ); insert_into_cache(&db_cache, &cb2); @@ -347,7 +349,7 @@ mod tests { validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -373,6 +375,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), + 0, ); let (cb2, _) = fake_compact_block( sapling_activation_height() + 1, @@ -380,12 +383,13 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(7).unwrap(), + 1, ); insert_into_cache(&db_cache, &cb); insert_into_cache(&db_cache, &cb2); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -397,6 +401,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(8).unwrap(), + 2, ); let (cb4, _) = fake_compact_block( sapling_activation_height() + 3, @@ -404,6 +409,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(3).unwrap(), + 3, ); insert_into_cache(&db_cache, &cb3); insert_into_cache(&db_cache, &cb4); @@ -434,6 +440,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), + 0, ); let (cb2, _) = fake_compact_block( sapling_activation_height() + 1, @@ -441,12 +448,13 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(7).unwrap(), + 1, ); insert_into_cache(&db_cache, &cb); insert_into_cache(&db_cache, &cb2); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -458,6 +466,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(8).unwrap(), + 2, ); let (cb4, _) = fake_compact_block( sapling_activation_height() + 3, @@ -465,6 +474,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(3).unwrap(), + 3, ); insert_into_cache(&db_cache, &cb3); insert_into_cache(&db_cache, &cb4); @@ -503,6 +513,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); let (cb2, _) = fake_compact_block( @@ -511,12 +522,13 @@ mod tests { &dfvk, AddressType::DefaultExternal, value2, + 1, ); insert_into_cache(&db_cache, &cb); insert_into_cache(&db_cache, &cb2); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should reflect both received notes assert_eq!( @@ -551,7 +563,7 @@ mod tests { ); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should again reflect both received notes assert_eq!( @@ -581,9 +593,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb1); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -596,6 +609,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 1, ); let (cb3, _) = fake_compact_block( sapling_activation_height() + 2, @@ -603,9 +617,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 2, ); insert_into_cache(&db_cache, &cb3); - match scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None) { + match scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None) { Err(Error::Chain(e)) => { assert_matches!( e.cause(), @@ -618,7 +633,7 @@ mod tests { // If we add a block of height SAPLING_ACTIVATION_HEIGHT + 1, we can now scan both insert_into_cache(&db_cache, &cb2); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), Amount::from_u64(150_000).unwrap() @@ -652,11 +667,12 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should reflect the received note assert_eq!( @@ -672,11 +688,12 @@ mod tests { &dfvk, AddressType::DefaultExternal, value2, + 1, ); insert_into_cache(&db_cache, &cb2); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should reflect both received notes assert_eq!( @@ -712,11 +729,12 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should reflect the received note assert_eq!( @@ -737,11 +755,12 @@ mod tests { &dfvk, to2, value2, + 1, ), ); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should equal the change assert_eq!( diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index cfd326b9c5..3f1832411b 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -3,6 +3,7 @@ use std::error; use std::fmt; +use shardtree::ShardTreeError; use zcash_client_backend::encoding::{Bech32DecodeError, TransparentCodecError}; use zcash_primitives::{consensus::BlockHeight, zip32::AccountId}; @@ -74,6 +75,9 @@ pub enum SqliteClientError { /// belonging to the wallet #[cfg(feature = "transparent-inputs")] AddressNotRecognized(TransparentAddress), + + /// An error occurred in inserting data into one of the wallet's note commitment trees. + CommitmentTree(ShardTreeError), } impl error::Error for SqliteClientError { @@ -114,6 +118,7 @@ impl fmt::Display for SqliteClientError { SqliteClientError::AccountIdOutOfRange => write!(f, "Wallet account identifiers must be less than 0x7FFFFFFF."), #[cfg(feature = "transparent-inputs")] SqliteClientError::AddressNotRecognized(_) => write!(f, "The address associated with a received txo is not identifiable as belonging to the wallet."), + SqliteClientError::CommitmentTree(err) => write!(f, "An error occurred accessing or updating note commitment tree data: {}.", err), } } } @@ -160,3 +165,9 @@ impl From for SqliteClientError { SqliteClientError::InvalidMemo(e) } } + +impl From> for SqliteClientError { + fn from(e: ShardTreeError) -> Self { + SqliteClientError::CommitmentTree(e) + } +} diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 1835b27a9c..e17cbab073 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -34,8 +34,10 @@ use rusqlite::{self, Connection}; use secrecy::{ExposeSecret, SecretVec}; -use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, path::Path}; +use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, ops::Range, path::Path}; +use incrementalmerkletree::Position; +use shardtree::{ShardTree, ShardTreeError}; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight}, @@ -52,8 +54,10 @@ use zcash_primitives::{ use zcash_client_backend::{ address::{AddressMetadata, UnifiedAddress}, data_api::{ - self, chain::BlockSource, DecryptedTransaction, NullifierQuery, PoolType, PrunedBlock, - Recipient, SentTransaction, WalletRead, WalletWrite, + self, + chain::{BlockSource, CommitmentTreeMeta}, + DecryptedTransaction, NullifierQuery, PoolType, PrunedBlock, Recipient, SentTransaction, + WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }, keys::{UnifiedFullViewingKey, UnifiedSpendingKey}, proto::compact_formats::CompactBlock, @@ -61,7 +65,9 @@ use zcash_client_backend::{ DecryptedOutput, TransferType, }; -use crate::error::SqliteClientError; +use crate::{ + error::SqliteClientError, wallet::sapling::commitment_tree::WalletDbSaplingShardStore, +}; #[cfg(feature = "unstable")] use { @@ -125,15 +131,15 @@ impl WalletDb { }) } - pub fn transactionally(&mut self, f: F) -> Result + pub fn transactionally>(&mut self, f: F) -> Result where - F: FnOnce(&WalletDb, P>) -> Result, + F: FnOnce(&mut WalletDb, P>) -> Result, { - let wdb = WalletDb { + let mut wdb = WalletDb { conn: SqlTransaction(self.conn.transaction()?), params: self.params.clone(), }; - let result = f(&wdb)?; + let result = f(&mut wdb)?; wdb.conn.0.commit()?; Ok(result) } @@ -148,6 +154,20 @@ impl, P: consensus::Parameters> WalletRead for W wallet::block_height_extrema(self.conn.borrow()).map_err(SqliteClientError::from) } + fn fully_scanned_height( + &self, + ) -> Result, Self::Error> { + wallet::fully_scanned_height(self.conn.borrow()) + } + + fn suggest_scan_ranges( + &self, + _batch_size: usize, + _limit: usize, + ) -> Result>, Self::Error> { + todo!() + } + fn get_min_unspent_height(&self) -> Result, Self::Error> { wallet::get_min_unspent_height(self.conn.borrow()).map_err(SqliteClientError::from) } @@ -210,24 +230,9 @@ impl, P: consensus::Parameters> WalletRead for W } } - fn get_commitment_tree( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error> { - wallet::sapling::get_sapling_commitment_tree(self.conn.borrow(), block_height) - } - - #[allow(clippy::type_complexity)] - fn get_witnesses( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error> { - wallet::sapling::get_sapling_witnesses(self.conn.borrow(), block_height) - } - fn get_sapling_nullifiers( &self, - query: data_api::NullifierQuery, + query: NullifierQuery, ) -> Result, Self::Error> { match query { NullifierQuery::Unspent => wallet::sapling::get_sapling_nullifiers(self.conn.borrow()), @@ -386,21 +391,21 @@ impl WalletWrite for WalletDb #[allow(clippy::type_complexity)] fn advance_by_block( &mut self, - block: &PrunedBlock, - updated_witnesses: &[(Self::NoteRef, sapling::IncrementalWitness)], - ) -> Result, Self::Error> { + block: PrunedBlock, + ) -> Result, Self::Error> { self.transactionally(|wdb| { // Insert the block into the database. + let block_height = block.block_height; wallet::insert_block( &wdb.conn.0, - block.block_height, + block_height, block.block_hash, block.block_time, - block.commitment_tree, + block.sapling_commitment_tree_size.map(|s| s.into()), )?; - let mut new_witnesses = vec![]; - for tx in block.transactions { + let mut wallet_note_ids = vec![]; + for tx in &block.transactions { let tx_row = wallet::put_tx_meta(&wdb.conn.0, tx, block.block_height)?; // Mark notes as spent and remove them from the scanning cache @@ -413,32 +418,24 @@ impl WalletWrite for WalletDb wallet::sapling::put_received_note(&wdb.conn.0, output, tx_row)?; // Save witness for note. - new_witnesses.push((received_note_id, output.witness().clone())); + wallet_note_ids.push(received_note_id); } } - // Insert current new_witnesses into the database. - for (received_note_id, witness) in updated_witnesses.iter().chain(new_witnesses.iter()) - { - if let NoteId::ReceivedNoteId(rnid) = *received_note_id { - wallet::sapling::insert_witness( - &wdb.conn.0, - rnid, - witness, - block.block_height, - )?; - } else { - return Err(SqliteClientError::InvalidNoteId); + let mut sapling_commitments = block.sapling_commitments.into_iter(); + wdb.with_sapling_tree_mut::<_, _, SqliteClientError>(move |sapling_tree| { + if let Some(sapling_tree_size) = block.sapling_commitment_tree_size { + let start_position = Position::from(u64::from(sapling_tree_size)) + - u64::try_from(sapling_commitments.len()).unwrap(); + sapling_tree.batch_insert(start_position, &mut sapling_commitments)?; } - } - - // Prune the stored witnesses (we only expect rollbacks of at most PRUNING_HEIGHT blocks). - wallet::prune_witnesses(&wdb.conn.0, block.block_height - PRUNING_HEIGHT)?; + Ok(()) + })?; // Update now-expired transactions that didn't get mined. - wallet::update_expired_notes(&wdb.conn.0, block.block_height)?; + wallet::update_expired_notes(&wdb.conn.0, block_height)?; - Ok(new_witnesses) + Ok(wallet_note_ids) }) } @@ -493,55 +490,37 @@ impl WalletWrite for WalletDb wallet::sapling::put_received_note(&wdb.conn.0, output, tx_ref)?; } } - } - // If any of the utxos spent in the transaction are ours, mark them as spent. - #[cfg(feature = "transparent-inputs")] - for txin in d_tx - .tx - .transparent_bundle() - .iter() - .flat_map(|b| b.vin.iter()) - { - wallet::mark_transparent_utxo_spent(&wdb.conn.0, tx_ref, &txin.prevout)?; - } + // If any of the utxos spent in the transaction are ours, mark them as spent. + #[cfg(feature = "transparent-inputs")] + for txin in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vin.iter()) { + wallet::mark_transparent_utxo_spent(&wdb.conn.0, tx_ref, &txin.prevout)?; + } - // If we have some transparent outputs: - if !d_tx - .tx - .transparent_bundle() - .iter() - .any(|b| b.vout.is_empty()) - { - let nullifiers = wdb.get_sapling_nullifiers(data_api::NullifierQuery::All)?; - // If the transaction contains shielded spends from our wallet, we will store z->t - // transactions we observe in the same way they would be stored by - // create_spend_to_address. - if let Some((account_id, _)) = nullifiers.iter().find(|(_, nf)| { - d_tx.tx - .sapling_bundle() - .iter() - .flat_map(|b| b.shielded_spends().iter()) - .any(|input| nf == input.nullifier()) - }) { - for (output_index, txout) in d_tx - .tx - .transparent_bundle() - .iter() - .flat_map(|b| b.vout.iter()) - .enumerate() - { - if let Some(address) = txout.recipient_address() { - wallet::put_sent_output( - &wdb.conn.0, - &wdb.params, - *account_id, - tx_ref, - output_index, - &Recipient::Transparent(address), - txout.value, - None, - )?; + // If we have some transparent outputs: + if d_tx.tx.transparent_bundle().iter().any(|b| !b.vout.is_empty()) { + let nullifiers = wdb.get_sapling_nullifiers(NullifierQuery::All)?; + // If the transaction contains shielded spends from our wallet, we will store z->t + // transactions we observe in the same way they would be stored by + // create_spend_to_address. + if let Some((account_id, _)) = nullifiers.iter().find( + |(_, nf)| + d_tx.tx.sapling_bundle().iter().flat_map(|b| b.shielded_spends().iter()) + .any(|input| nf == input.nullifier()) + ) { + for (output_index, txout) in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vout.iter()).enumerate() { + if let Some(address) = txout.recipient_address() { + wallet::put_sent_output( + &wdb.conn.0, + &wdb.params, + *account_id, + tx_ref, + output_index, + &Recipient::Transparent(address), + txout.value, + None + )?; + } } } } @@ -633,6 +612,59 @@ impl WalletWrite for WalletDb } } +impl WalletCommitmentTrees for WalletDb { + type Error = rusqlite::Error; + type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; + + fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::SaplingShardStore<'a>, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, + ) -> Result, + E: From>, + { + let tx = self.conn.transaction().map_err(ShardTreeError::Storage)?; + let shard_store = + WalletDbSaplingShardStore::from_connection(&tx).map_err(ShardTreeError::Storage)?; + let result = { + let mut shardtree = ShardTree::new(shard_store, 100); + callback(&mut shardtree)? + }; + tx.commit().map_err(ShardTreeError::Storage)?; + Ok(result) + } +} + +impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb, P> { + type Error = rusqlite::Error; + type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; + + fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::SaplingShardStore<'a>, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, + ) -> Result, + E: From>, + { + let mut shardtree = ShardTree::new( + WalletDbSaplingShardStore::from_connection(&self.conn.0) + .map_err(ShardTreeError::Storage)?, + 100, + ); + let result = callback(&mut shardtree)?; + + Ok(result) + } +} + /// A handle for the SQLite block source. pub struct BlockDb(Connection); @@ -1024,6 +1056,7 @@ mod tests { dfvk: &DiversifiableFullViewingKey, req: AddressType, value: Amount, + initial_sapling_tree_size: u32, ) -> (CompactBlock, Nullifier) { let to = match req { AddressType::DefaultExternal => dfvk.default_address().1, @@ -1069,6 +1102,8 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); + cb.sapling_commitment_tree_size = initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); (cb, note.nf(&dfvk.fvk().vk.nk, 0)) } @@ -1081,6 +1116,7 @@ mod tests { dfvk: &DiversifiableFullViewingKey, to: PaymentAddress, value: Amount, + initial_sapling_tree_size: u32, ) -> CompactBlock { let mut rng = OsRng; let rseed = generate_random_rseed(&network(), height, &mut rng); @@ -1154,6 +1190,8 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); + cb.sapling_commitment_tree_size = initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); cb } @@ -1267,6 +1305,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), + 0, ); let (cb2, _) = fake_compact_block( BlockHeight::from_u32(2), @@ -1274,6 +1313,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(10).unwrap(), + 1, ); // Write the CompactBlocks to the BlockMeta DB's corresponding disk storage. diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index a05597134f..406db08c76 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -67,13 +67,13 @@ use rusqlite::{self, named_params, params, OptionalExtension, ToSql}; use std::collections::HashMap; use std::convert::TryFrom; +use std::io::Cursor; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight, BranchId, NetworkUpgrade, Parameters}, memo::{Memo, MemoBytes}, - merkle_tree::write_commitment_tree, - sapling::CommitmentTree, + merkle_tree::read_commitment_tree, transaction::{components::Amount, Transaction, TxId}, zip32::{ sapling::{DiversifiableFullViewingKey, ExtendedFullViewingKey}, @@ -83,7 +83,7 @@ use zcash_primitives::{ use zcash_client_backend::{ address::{RecipientAddress, UnifiedAddress}, - data_api::{PoolType, Recipient, SentTransactionOutput}, + data_api::{chain::CommitmentTreeMeta, PoolType, Recipient, SentTransactionOutput}, encoding::AddressCodec, keys::UnifiedFullViewingKey, wallet::WalletTx, @@ -536,6 +536,51 @@ pub(crate) fn block_height_extrema( }) } +pub(crate) fn fully_scanned_height( + conn: &rusqlite::Connection, +) -> Result, SqliteClientError> { + let res_opt = conn + .query_row( + "SELECT height, sapling_commitment_tree_size, sapling_tree + FROM blocks + ORDER BY height DESC + LIMIT 1", + [], + |row| { + let max_height: u32 = row.get(0)?; + let sapling_tree_size: Option = row.get(1)?; + let sapling_tree: Vec = row.get(0)?; + Ok(( + BlockHeight::from(max_height), + sapling_tree_size, + sapling_tree, + )) + }, + ) + .optional()?; + + res_opt + .map(|(max_height, sapling_tree_size, sapling_tree)| { + let commitment_tree_meta = + CommitmentTreeMeta::from_parts(if let Some(known_size) = sapling_tree_size { + known_size + } else { + // parse the legacy commitment tree data + read_commitment_tree::< + zcash_primitives::sapling::Node, + _, + { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, + >(Cursor::new(sapling_tree))? + .size() + .try_into() + .expect("usize values are convertible to u64 on all supported platforms.") + }); + + Ok((max_height, commitment_tree_meta)) + }) + .transpose() +} + /// Returns the block height at which the specified transaction was mined, /// if any. pub(crate) fn get_tx_height( @@ -765,21 +810,24 @@ pub(crate) fn insert_block( block_height: BlockHeight, block_hash: BlockHash, block_time: u32, - commitment_tree: &CommitmentTree, + sapling_commitment_tree_size: Option, ) -> Result<(), SqliteClientError> { - let mut encoded_tree = Vec::new(); - write_commitment_tree(commitment_tree, &mut encoded_tree).unwrap(); - let mut stmt_insert_block = conn.prepare_cached( - "INSERT INTO blocks (height, hash, time, sapling_tree) - VALUES (?, ?, ?, ?)", + "INSERT INTO blocks ( + height, + hash, + time, + sapling_commitment_tree_size, + sapling_tree + ) + VALUES (?, ?, ?, ?, x'00')", )?; stmt_insert_block.execute(params![ u32::from(block_height), &block_hash.0[..], block_time, - encoded_tree + sapling_commitment_tree_size ])?; Ok(()) @@ -951,17 +999,6 @@ pub(crate) fn put_legacy_transparent_utxo( stmt_upsert_legacy_transparent_utxo.query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId)) } -/// Removes old incremental witnesses up to the given block height. -pub(crate) fn prune_witnesses( - conn: &rusqlite::Connection, - below_height: BlockHeight, -) -> Result<(), SqliteClientError> { - let mut stmt_prune_witnesses = - conn.prepare_cached("DELETE FROM sapling_witnesses WHERE block < ?")?; - stmt_prune_witnesses.execute([u32::from(below_height)])?; - Ok(()) -} - /// Marks notes that have not been mined in transactions /// as expired, up to the given block height. pub(crate) fn update_expired_notes( diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 7f5a60ccc4..bb8834a6a2 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -6,6 +6,7 @@ use rusqlite::{self, types::ToSql}; use schemer::{Migrator, MigratorError}; use schemer_rusqlite::RusqliteAdapter; use secrecy::SecretVec; +use shardtree::ShardTreeError; use uuid::Uuid; use zcash_primitives::{ @@ -34,6 +35,9 @@ pub enum WalletMigrationError { /// Wrapper for amount balance violations BalanceError(BalanceError), + + /// Wrapper for commitment tree invariant violations + CommitmentTree(ShardTreeError), } impl From for WalletMigrationError { @@ -48,6 +52,12 @@ impl From for WalletMigrationError { } } +impl From> for WalletMigrationError { + fn from(e: ShardTreeError) -> Self { + WalletMigrationError::CommitmentTree(e) + } +} + impl fmt::Display for WalletMigrationError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match &self { @@ -62,6 +72,7 @@ impl fmt::Display for WalletMigrationError { } WalletMigrationError::DbError(e) => write!(f, "{}", e), WalletMigrationError::BalanceError(e) => write!(f, "Balance error: {:?}", e), + WalletMigrationError::CommitmentTree(e) => write!(f, "Commitment tree error: {:?}", e), } } } @@ -361,8 +372,9 @@ mod tests { height INTEGER PRIMARY KEY, hash BLOB NOT NULL, time INTEGER NOT NULL, - sapling_tree BLOB NOT NULL - )", + sapling_tree BLOB NOT NULL , + sapling_commitment_tree_size INTEGER, + orchard_commitment_tree_size INTEGER)", "CREATE TABLE sapling_received_notes ( id_note INTEGER PRIMARY KEY, tx INTEGER NOT NULL, @@ -375,6 +387,7 @@ mod tests { is_change INTEGER NOT NULL, memo BLOB, spent INTEGER, + commitment_tree_position INTEGER, FOREIGN KEY (tx) REFERENCES transactions(id_tx), FOREIGN KEY (account) REFERENCES accounts(account), FOREIGN KEY (spent) REFERENCES transactions(id_tx), diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 1cc9bcfc5f..e51605ccf4 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -4,6 +4,7 @@ mod addresses_table; mod initial_setup; mod received_notes_nullable_nf; mod sent_notes_to_internal; +mod shardtree_support; mod ufvk_support; mod utxos_table; mod v_transactions_net; @@ -46,5 +47,6 @@ pub(super) fn all_migrations( Box::new(add_transaction_views::Migration), Box::new(v_transactions_net::Migration), Box::new(received_notes_nullable_nf::Migration), + Box::new(shardtree_support::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs new file mode 100644 index 0000000000..b9e4a6bf04 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -0,0 +1,56 @@ +//! This migration adds tables to the wallet database that are needed to persist note commitment +//! tree data using the `shardtree` crate, and migrates existing witness data into these data +//! structures. + +use std::collections::HashSet; + +use rusqlite; +use schemer; +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::{migrations::received_notes_nullable_nf, WalletMigrationError}; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( + 0x7da6489d, + 0xe835, + 0x4657, + b"\x8b\xe5\xf5\x12\xbc\xce\x6c\xbf", +); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [received_notes_nullable_nf::MIGRATION_ID] + .into_iter() + .collect() + } + + fn description(&self) -> &'static str { + "Add support for receiving storage of note commitment tree data using the `shardtree` crate." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + // Add commitment tree sizes to block metadata. + transaction.execute_batch( + "ALTER TABLE blocks ADD COLUMN sapling_commitment_tree_size INTEGER; + ALTER TABLE blocks ADD COLUMN orchard_commitment_tree_size INTEGER; + ALTER TABLE sapling_received_notes ADD COLUMN commitment_tree_position INTEGER;", + )?; + + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + Ok(()) + } +} diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 66734cfdc3..511333ff7f 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -1,12 +1,13 @@ //! Functions for Sapling support in the wallet. + use group::ff::PrimeField; -use rusqlite::{named_params, params, types::Value, Connection, OptionalExtension, Row}; +use incrementalmerkletree::Position; +use rusqlite::{named_params, params, types::Value, Connection, Row}; use std::rc::Rc; use zcash_primitives::{ consensus::BlockHeight, memo::MemoBytes, - merkle_tree::{read_commitment_tree, read_incremental_witness, write_incremental_witness}, sapling::{self, Diversifier, Note, Nullifier, Rseed}, transaction::components::Amount, zip32::AccountId, @@ -21,6 +22,8 @@ use crate::{error::SqliteClientError, NoteId}; use super::memo_repr; +pub(crate) mod commitment_tree; + /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedSaplingOutput { fn index(&self) -> usize; @@ -28,10 +31,11 @@ pub(crate) trait ReceivedSaplingOutput { fn note(&self) -> &Note; fn memo(&self) -> Option<&MemoBytes>; fn is_change(&self) -> bool; - fn nullifier(&self) -> Option<&Nullifier>; + fn nullifier(&self) -> Option<&sapling::Nullifier>; + fn note_commitment_tree_position(&self) -> Option; } -impl ReceivedSaplingOutput for WalletSaplingOutput { +impl ReceivedSaplingOutput for WalletSaplingOutput { fn index(&self) -> usize { self.index() } @@ -47,10 +51,12 @@ impl ReceivedSaplingOutput for WalletSaplingOutput { fn is_change(&self) -> bool { WalletSaplingOutput::is_change(self) } - - fn nullifier(&self) -> Option<&Nullifier> { + fn nullifier(&self) -> Option<&sapling::Nullifier> { Some(self.nf()) } + fn note_commitment_tree_position(&self) -> Option { + Some(WalletSaplingOutput::note_commitment_tree_position(self)) + } } impl ReceivedSaplingOutput for DecryptedOutput { @@ -69,7 +75,10 @@ impl ReceivedSaplingOutput for DecryptedOutput { fn is_change(&self) -> bool { self.transfer_type == TransferType::WalletInternal } - fn nullifier(&self) -> Option<&Nullifier> { + fn nullifier(&self) -> Option<&sapling::Nullifier> { + None + } + fn note_commitment_tree_position(&self) -> Option { None } } @@ -105,17 +114,17 @@ fn to_spendable_note(row: &Row) -> Result, SqliteCli Rseed::BeforeZip212(rcm) }; - let witness = { - let d: Vec<_> = row.get(4)?; - read_incremental_witness(&d[..])? - }; + let note_commitment_tree_position = + Position::from(u64::try_from(row.get::<_, i64>(4)?).map_err(|_| { + SqliteClientError::CorruptedData("Note commitment tree position invalid.".to_string()) + })?); Ok(ReceivedSaplingNote { note_id, diversifier, note_value, rseed, - witness, + note_commitment_tree_position, }) } @@ -126,15 +135,13 @@ pub(crate) fn get_spendable_sapling_notes( exclude: &[NoteId], ) -> Result>, SqliteClientError> { let mut stmt_select_notes = conn.prepare_cached( - "SELECT id_note, diversifier, value, rcm, witness - FROM sapling_received_notes - INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx - INNER JOIN sapling_witnesses ON sapling_witnesses.note = sapling_received_notes.id_note - WHERE account = :account - AND spent IS NULL - AND transactions.block <= :anchor_height - AND sapling_witnesses.block = :anchor_height - AND id_note NOT IN rarray(:exclude)", + "SELECT id_note, diversifier, value, rcm, commitment_tree_position + FROM sapling_received_notes + INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx + WHERE account = :account + AND spent IS NULL + AND transactions.block <= :anchor_height + AND id_note NOT IN rarray(:exclude)", )?; let excluded: Vec = exclude @@ -184,28 +191,22 @@ pub(crate) fn select_spendable_sapling_notes( // // 4) Match the selected notes against the witnesses at the desired height. let mut stmt_select_notes = conn.prepare_cached( - "WITH selected AS ( - WITH eligible AS ( - SELECT id_note, diversifier, value, rcm, - SUM(value) OVER - (PARTITION BY account, spent ORDER BY id_note) AS so_far - FROM sapling_received_notes - INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx - WHERE account = :account - AND spent IS NULL - AND transactions.block <= :anchor_height - AND id_note NOT IN rarray(:exclude) - ) - SELECT * FROM eligible WHERE so_far < :target_value - UNION - SELECT * FROM (SELECT * FROM eligible WHERE so_far >= :target_value LIMIT 1) - ), witnesses AS ( - SELECT note, witness FROM sapling_witnesses - WHERE block = :anchor_height - ) - SELECT selected.id_note, selected.diversifier, selected.value, selected.rcm, witnesses.witness - FROM selected - INNER JOIN witnesses ON selected.id_note = witnesses.note", + "WITH eligible AS ( + SELECT id_note, diversifier, value, rcm, commitment_tree_position, + SUM(value) + OVER (PARTITION BY account, spent ORDER BY id_note) AS so_far + FROM sapling_received_notes + INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx + WHERE account = :account + AND spent IS NULL + AND transactions.block <= :anchor_height + AND id_note NOT IN rarray(:exclude) + ) + SELECT id_note, diversifier, value, rcm, commitment_tree_position + FROM eligible WHERE so_far < :target_value + UNION + SELECT id_note, diversifier, value, rcm, commitment_tree_position + FROM (SELECT * from eligible WHERE so_far >= :target_value LIMIT 1)", )?; let excluded: Vec = exclude @@ -230,73 +231,6 @@ pub(crate) fn select_spendable_sapling_notes( notes.collect::>() } -/// Returns the commitment tree for the block at the specified height, -/// if any. -pub(crate) fn get_sapling_commitment_tree( - conn: &Connection, - block_height: BlockHeight, -) -> Result, SqliteClientError> { - conn.query_row_and_then( - "SELECT sapling_tree FROM blocks WHERE height = ?", - [u32::from(block_height)], - |row| { - let row_data: Vec = row.get(0)?; - read_commitment_tree(&row_data[..]).map_err(|e| { - rusqlite::Error::FromSqlConversionFailure( - row_data.len(), - rusqlite::types::Type::Blob, - Box::new(e), - ) - }) - }, - ) - .optional() - .map_err(SqliteClientError::from) -} - -/// Returns the incremental witnesses for the block at the specified height, -/// if any. -pub(crate) fn get_sapling_witnesses( - conn: &Connection, - block_height: BlockHeight, -) -> Result, SqliteClientError> { - let mut stmt_fetch_witnesses = - conn.prepare_cached("SELECT note, witness FROM sapling_witnesses WHERE block = ?")?; - - let witnesses = stmt_fetch_witnesses - .query_map([u32::from(block_height)], |row| { - let id_note = NoteId::ReceivedNoteId(row.get(0)?); - let witness_data: Vec = row.get(1)?; - Ok(read_incremental_witness(&witness_data[..]).map(|witness| (id_note, witness))) - }) - .map_err(SqliteClientError::from)?; - - // unwrap database error & IO error from IncrementalWitness::read - let res: Vec<_> = witnesses.collect::, _>>()??; - Ok(res) -} - -/// Records the incremental witness for the specified note, -/// as of the given block height. -pub(crate) fn insert_witness( - conn: &Connection, - note_id: i64, - witness: &sapling::IncrementalWitness, - height: BlockHeight, -) -> Result<(), SqliteClientError> { - let mut stmt_insert_witness = conn.prepare_cached( - "INSERT INTO sapling_witnesses (note, block, witness) - VALUES (?, ?, ?)", - )?; - - let mut encoded = Vec::new(); - write_incremental_witness(witness, &mut encoded).unwrap(); - - stmt_insert_witness.execute(params![note_id, u32::from(height), encoded])?; - - Ok(()) -} - /// Retrieves the set of nullifiers for "potentially spendable" Sapling notes that the /// wallet is tracking. /// @@ -320,7 +254,7 @@ pub(crate) fn get_sapling_nullifiers( let nf_bytes: Vec = row.get(2)?; Ok(( AccountId::from(account), - Nullifier::from_slice(&nf_bytes).unwrap(), + sapling::Nullifier::from_slice(&nf_bytes).unwrap(), )) })?; @@ -343,7 +277,7 @@ pub(crate) fn get_all_sapling_nullifiers( let nf_bytes: Vec = row.get(2)?; Ok(( AccountId::from(account), - Nullifier::from_slice(&nf_bytes).unwrap(), + sapling::Nullifier::from_slice(&nf_bytes).unwrap(), )) })?; @@ -359,7 +293,7 @@ pub(crate) fn get_all_sapling_nullifiers( pub(crate) fn mark_sapling_note_spent( conn: &Connection, tx_ref: i64, - nf: &Nullifier, + nf: &sapling::Nullifier, ) -> Result { let mut stmt_mark_sapling_note_spent = conn.prepare_cached("UPDATE sapling_received_notes SET spent = ? WHERE nf = ?")?; @@ -383,9 +317,19 @@ pub(crate) fn put_received_note( ) -> Result { let mut stmt_upsert_received_note = conn.prepare_cached( "INSERT INTO sapling_received_notes - (tx, output_index, account, diversifier, value, rcm, memo, nf, is_change) - VALUES - (:tx, :output_index, :account, :diversifier, :value, :rcm, :memo, :nf, :is_change) + (tx, output_index, account, diversifier, value, rcm, memo, nf, is_change, commitment_tree_position) + VALUES ( + :tx, + :output_index, + :account, + :diversifier, + :value, + :rcm, + :memo, + :nf, + :is_change, + :commitment_tree_position + ) ON CONFLICT (tx, output_index) DO UPDATE SET account = :account, diversifier = :diversifier, @@ -393,7 +337,8 @@ pub(crate) fn put_received_note( rcm = :rcm, nf = IFNULL(:nf, nf), memo = IFNULL(:memo, memo), - is_change = IFNULL(:is_change, is_change) + is_change = IFNULL(:is_change, is_change), + commitment_tree_position = IFNULL(:commitment_tree_position, commitment_tree_position) RETURNING id_note", )?; @@ -410,7 +355,8 @@ pub(crate) fn put_received_note( ":rcm": &rcm.as_ref(), ":nf": output.nullifier().map(|nf| nf.0.as_ref()), ":memo": memo_repr(output.memo()), - ":is_change": output.is_change() + ":is_change": output.is_change(), + ":commitment_tree_position": output.note_commitment_tree_position().map(u64::from), ]; stmt_upsert_received_note @@ -622,9 +568,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); @@ -644,9 +591,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 1, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance does not include the second note let (_, anchor_height2) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); @@ -691,10 +639,11 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + i, ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Second spend still fails assert_matches!( @@ -724,9 +673,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 11, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Second spend should now succeed assert_matches!( @@ -768,9 +718,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -823,10 +774,11 @@ mod tests { &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, + i, ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Second spend still fails assert_matches!( @@ -855,9 +807,10 @@ mod tests { &ExtendedSpendingKey::master(&[42]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, + 42, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Second spend should now succeed create_spend_to_address( @@ -898,9 +851,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -968,10 +922,11 @@ mod tests { &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, + i, ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&network, &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&network, &db_cache, &mut db_data, None, None).unwrap(); // Send the funds again, discarding history. // Neither transaction output is decryptable by the sender. @@ -1001,9 +956,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); @@ -1056,9 +1012,10 @@ mod tests { &dfvk, AddressType::Internal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); @@ -1110,6 +1067,7 @@ mod tests { &dfvk, AddressType::Internal, Amount::from_u64(50000).unwrap(), + 0, ); insert_into_cache(&db_cache, &cb); @@ -1121,11 +1079,12 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(1000).unwrap(), + i, ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance let total = Amount::from_u64(60000).unwrap(); @@ -1241,9 +1200,10 @@ mod tests { &dfvk, AddressType::Internal, Amount::from_u64(50000).unwrap(), + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_matches!( shield_transparent_funds( diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs new file mode 100644 index 0000000000..a02ce4cd8d --- /dev/null +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -0,0 +1,123 @@ +use incrementalmerkletree::Address; +use rusqlite; +use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore}; + +use zcash_primitives::{consensus::BlockHeight, sapling}; + +pub struct WalletDbSaplingShardStore<'conn, 'a> { + pub(crate) conn: &'a rusqlite::Transaction<'conn>, +} + +impl<'conn, 'a> WalletDbSaplingShardStore<'conn, 'a> { + pub(crate) fn from_connection( + conn: &'a rusqlite::Transaction<'conn>, + ) -> Result { + Ok(WalletDbSaplingShardStore { conn }) + } +} + +impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { + type H = sapling::Node; + type CheckpointId = BlockHeight; + type Error = rusqlite::Error; + + fn get_shard( + &self, + _shard_root: Address, + ) -> Result>, Self::Error> { + // SELECT shard_data FROM sapling_tree WHERE shard_index = shard_root.index + todo!() + } + + fn last_shard(&self) -> Result>, Self::Error> { + // SELECT shard_data FROM sapling_tree ORDER BY shard_index DESC LIMIT 1 + todo!() + } + + fn put_shard(&mut self, _subtree: LocatedPrunableTree) -> Result<(), Self::Error> { + todo!() + } + + fn get_shard_roots(&self) -> Result, Self::Error> { + // SELECT + todo!() + } + + fn truncate(&mut self, _from: Address) -> Result<(), Self::Error> { + todo!() + } + + fn get_cap(&self) -> Result, Self::Error> { + todo!() + } + + fn put_cap(&mut self, _cap: PrunableTree) -> Result<(), Self::Error> { + todo!() + } + + fn min_checkpoint_id(&self) -> Result, Self::Error> { + todo!() + } + + fn max_checkpoint_id(&self) -> Result, Self::Error> { + todo!() + } + + fn add_checkpoint( + &mut self, + _checkpoint_id: Self::CheckpointId, + _checkpoint: Checkpoint, + ) -> Result<(), Self::Error> { + todo!() + } + + fn checkpoint_count(&self) -> Result { + todo!() + } + + fn get_checkpoint_at_depth( + &self, + _checkpoint_depth: usize, + ) -> Result, Self::Error> { + todo!() + } + + fn get_checkpoint( + &self, + _checkpoint_id: &Self::CheckpointId, + ) -> Result, Self::Error> { + todo!() + } + + fn with_checkpoints(&mut self, _limit: usize, _callback: F) -> Result<(), Self::Error> + where + F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, + { + todo!() + } + + fn update_checkpoint_with( + &mut self, + _checkpoint_id: &Self::CheckpointId, + _update: F, + ) -> Result + where + F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, + { + todo!() + } + + fn remove_checkpoint( + &mut self, + _checkpoint_id: &Self::CheckpointId, + ) -> Result<(), Self::Error> { + todo!() + } + + fn truncate_checkpoints( + &mut self, + _checkpoint_id: &Self::CheckpointId, + ) -> Result<(), Self::Error> { + todo!() + } +} diff --git a/zcash_primitives/src/merkle_tree.rs b/zcash_primitives/src/merkle_tree.rs index 176d3b4375..6cda449bcc 100644 --- a/zcash_primitives/src/merkle_tree.rs +++ b/zcash_primitives/src/merkle_tree.rs @@ -98,7 +98,7 @@ pub fn write_nonempty_frontier_v1( frontier: &NonEmptyFrontier, ) -> io::Result<()> { write_position(&mut writer, frontier.position())?; - if frontier.position().is_odd() { + if frontier.position().is_right_child() { // The v1 serialization wrote the sibling of a right-hand leaf as an optional value, rather // than as part of the ommers vector. frontier From ed2e22b73724b0e95896cd7c13620ef91da555d1 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 24 May 2023 21:57:54 -0600 Subject: [PATCH 0023/1122] zcash_client_sqlite: Add shard serialization & parsing --- zcash_client_sqlite/Cargo.toml | 5 +- zcash_client_sqlite/src/lib.rs | 1 + zcash_client_sqlite/src/serialization.rs | 114 +++++++++++++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 zcash_client_sqlite/src/serialization.rs diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 88585e0d22..6e3621b1ab 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -17,6 +17,7 @@ rust-version = "1.65" incrementalmerkletree = { version = "0.4", features = ["legacy-api"] } shardtree = { version = "0.0", features = ["legacy-api"] } zcash_client_backend = { version = "0.9", path = "../zcash_client_backend" } +zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_primitives = { version = "0.12", path = "../zcash_primitives", default-features = false } # Dependencies exposed in a public API: @@ -28,7 +29,8 @@ hdwallet = { version = "0.4", optional = true } # - Logging and metrics tracing = "0.1" -# - Protobuf interfaces +# - Serialization +byteorder = "1" prost = "0.11" # - Secret management @@ -49,6 +51,7 @@ uuid = "1.1" [dev-dependencies] assert_matches = "1.5" incrementalmerkletree = { version = "0.4", features = ["legacy-api", "test-dependencies"] } +shardtree = { version = "0.0", features = ["legacy-api", "test-dependencies"] } proptest = "1.0.0" rand_core = "0.6" regex = "1.4" diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index e17cbab073..c9062fe94b 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -78,6 +78,7 @@ use { pub mod chain; pub mod error; +mod serialization; pub mod wallet; /// The maximum number of blocks the wallet is allowed to rewind. This is diff --git a/zcash_client_sqlite/src/serialization.rs b/zcash_client_sqlite/src/serialization.rs new file mode 100644 index 0000000000..99cb90dd82 --- /dev/null +++ b/zcash_client_sqlite/src/serialization.rs @@ -0,0 +1,114 @@ +//! Serialization formats for data stored as SQLite BLOBs + +use byteorder::{ReadBytesExt, WriteBytesExt}; +use core::ops::Deref; +use shardtree::{Node, PrunableTree, RetentionFlags, Tree}; +use std::io::{self, Read, Write}; +use std::rc::Rc; +use zcash_encoding::Optional; +use zcash_primitives::merkle_tree::HashSer; + +const SER_V1: u8 = 1; + +const NIL_TAG: u8 = 0; +const LEAF_TAG: u8 = 1; +const PARENT_TAG: u8 = 2; + +pub fn write_shard_v1( + writer: &mut W, + tree: &PrunableTree, +) -> io::Result<()> { + fn write_inner( + mut writer: &mut W, + tree: &PrunableTree, + ) -> io::Result<()> { + match tree.deref() { + Node::Parent { ann, left, right } => { + writer.write_u8(PARENT_TAG)?; + Optional::write(&mut writer, ann.as_ref(), |w, h| { + ::write(h, w) + })?; + write_inner(writer, left)?; + write_inner(writer, right)?; + Ok(()) + } + Node::Leaf { value } => { + writer.write_u8(LEAF_TAG)?; + value.0.write(&mut writer)?; + writer.write_u8(value.1.bits())?; + Ok(()) + } + Node::Nil => { + writer.write_u8(NIL_TAG)?; + Ok(()) + } + } + } + + writer.write_u8(SER_V1)?; + write_inner(writer, tree) +} + +fn read_shard_v1(mut reader: &mut R) -> io::Result> { + match reader.read_u8()? { + PARENT_TAG => { + let ann = Optional::read(&mut reader, ::read)?.map(Rc::new); + let left = read_shard_v1(reader)?; + let right = read_shard_v1(reader)?; + Ok(Tree::parent(ann, left, right)) + } + LEAF_TAG => { + let value = ::read(&mut reader)?; + let flags = reader.read_u8().and_then(|bits| { + RetentionFlags::from_bits(bits).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Byte value {} does not correspond to a valid set of retention flags", + bits + ), + ) + }) + })?; + Ok(Tree::leaf((value, flags))) + } + NIL_TAG => Ok(Tree::empty()), + other => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Node tag not recognized: {}", other), + )), + } +} + +pub fn read_shard(mut reader: R) -> io::Result> { + match reader.read_u8()? { + SER_V1 => read_shard_v1(&mut reader), + other => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Shard serialization version not recognized: {}", other), + )), + } +} + +#[cfg(test)] +mod tests { + use incrementalmerkletree::frontier::testing::{arb_test_node, TestNode}; + use proptest::prelude::*; + use shardtree::testing::arb_prunable_tree; + use std::io::Cursor; + + use super::{read_shard, write_shard_v1}; + + proptest! { + #[test] + fn check_shard_roundtrip( + tree in arb_prunable_tree(arb_test_node(), 8, 32) + ) { + let mut tree_data = vec![]; + write_shard_v1(&mut tree_data, &tree).unwrap(); + let cursor = Cursor::new(tree_data); + let tree_result = read_shard::(cursor).unwrap(); + assert_eq!(tree, tree_result); + } + } +} From 9f2bb94a5e30cd8d51c9ef358ce9246e6c6414c9 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 3 Apr 2023 13:53:43 -0600 Subject: [PATCH 0024/1122] zcash_client_sqlite: Add shard persistence to wallet migration. --- zcash_client_sqlite/Cargo.toml | 6 +- zcash_client_sqlite/src/error.rs | 8 +- zcash_client_sqlite/src/lib.rs | 29 ++-- zcash_client_sqlite/src/wallet/init.rs | 30 +++- .../init/migrations/shardtree_support.rs | 138 +++++++++++++++++- .../src/wallet/sapling/commitment_tree.rs | 33 ++++- 6 files changed, 213 insertions(+), 31 deletions(-) diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 6e3621b1ab..6a951ff108 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -32,13 +32,14 @@ tracing = "0.1" # - Serialization byteorder = "1" prost = "0.11" +either = "1.8" +group = "0.13" +jubjub = "0.10" # - Secret management secrecy = "0.8" # - SQLite databases -group = "0.13" -jubjub = "0.10" rusqlite = { version = "0.29.0", features = ["bundled", "time", "array"] } schemer = "0.2" schemer-rusqlite = "0.2.2" @@ -67,6 +68,7 @@ test-dependencies = [ "incrementalmerkletree/test-dependencies", "zcash_primitives/test-dependencies", "zcash_client_backend/test-dependencies", + "incrementalmerkletree/test-dependencies", ] transparent-inputs = ["hdwallet", "zcash_client_backend/transparent-inputs"] unstable = ["zcash_client_backend/unstable"] diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 3f1832411b..87f88b8441 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -1,7 +1,9 @@ //! Error types for problems that may arise when reading or storing wallet data to SQLite. +use either::Either; use std::error; use std::fmt; +use std::io; use shardtree::ShardTreeError; use zcash_client_backend::encoding::{Bech32DecodeError, TransparentCodecError}; @@ -77,7 +79,7 @@ pub enum SqliteClientError { AddressNotRecognized(TransparentAddress), /// An error occurred in inserting data into one of the wallet's note commitment trees. - CommitmentTree(ShardTreeError), + CommitmentTree(ShardTreeError>), } impl error::Error for SqliteClientError { @@ -166,8 +168,8 @@ impl From for SqliteClientError { } } -impl From> for SqliteClientError { - fn from(e: ShardTreeError) -> Self { +impl From>> for SqliteClientError { + fn from(e: ShardTreeError>) -> Self { SqliteClientError::CommitmentTree(e) } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index c9062fe94b..88b884b12f 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -32,9 +32,10 @@ // Catch documentation errors caused by code changes. #![deny(rustdoc::broken_intra_doc_links)] +use either::Either; use rusqlite::{self, Connection}; use secrecy::{ExposeSecret, SecretVec}; -use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, ops::Range, path::Path}; +use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, io, ops::Range, path::Path}; use incrementalmerkletree::Position; use shardtree::{ShardTree, ShardTreeError}; @@ -72,13 +73,13 @@ use crate::{ #[cfg(feature = "unstable")] use { crate::chain::{fsblockdb_with_blocks, BlockMeta}, + std::fs, std::path::PathBuf, - std::{fs, io}, }; pub mod chain; pub mod error; -mod serialization; +pub mod serialization; pub mod wallet; /// The maximum number of blocks the wallet is allowed to rewind. This is @@ -614,7 +615,7 @@ impl WalletWrite for WalletDb } impl WalletCommitmentTrees for WalletDb { - type Error = rusqlite::Error; + type Error = Either; type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result @@ -626,22 +627,26 @@ impl WalletCommitmentTrees for WalletDb, ) -> Result, - E: From>, + E: From>>, { - let tx = self.conn.transaction().map_err(ShardTreeError::Storage)?; - let shard_store = - WalletDbSaplingShardStore::from_connection(&tx).map_err(ShardTreeError::Storage)?; + let tx = self + .conn + .transaction() + .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; + let shard_store = WalletDbSaplingShardStore::from_connection(&tx) + .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; let result = { let mut shardtree = ShardTree::new(shard_store, 100); callback(&mut shardtree)? }; - tx.commit().map_err(ShardTreeError::Storage)?; + tx.commit() + .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; Ok(result) } } impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb, P> { - type Error = rusqlite::Error; + type Error = Either; type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result @@ -653,11 +658,11 @@ impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb, ) -> Result, - E: From>, + E: From>>, { let mut shardtree = ShardTree::new( WalletDbSaplingShardStore::from_connection(&self.conn.0) - .map_err(ShardTreeError::Storage)?, + .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?, 100, ); let result = callback(&mut shardtree)?; diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index bb8834a6a2..e3485c8ab8 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -1,6 +1,6 @@ //! Functions for initializing the various databases. -use std::collections::HashMap; -use std::fmt; +use either::Either; +use std::{collections::HashMap, fmt, io}; use rusqlite::{self, types::ToSql}; use schemer::{Migrator, MigratorError}; @@ -37,7 +37,7 @@ pub enum WalletMigrationError { BalanceError(BalanceError), /// Wrapper for commitment tree invariant violations - CommitmentTree(ShardTreeError), + CommitmentTree(ShardTreeError>), } impl From for WalletMigrationError { @@ -52,8 +52,8 @@ impl From for WalletMigrationError { } } -impl From> for WalletMigrationError { - fn from(e: ShardTreeError) -> Self { +impl From>> for WalletMigrationError { + fn from(e: ShardTreeError>) -> Self { WalletMigrationError::CommitmentTree(e) } } @@ -393,6 +393,26 @@ mod tests { FOREIGN KEY (spent) REFERENCES transactions(id_tx), CONSTRAINT tx_output UNIQUE (tx, output_index) )", + "CREATE TABLE sapling_tree_cap ( + cap_data BLOB NOT NULL + )", + "CREATE TABLE sapling_tree_checkpoint_marks_removed ( + checkpoint_id INTEGER NOT NULL, + mark_removed_position INTEGER NOT NULL, + FOREIGN KEY (checkpoint_id) REFERENCES sapling_tree_checkpoints(checkpoint_id) + )", + "CREATE TABLE sapling_tree_checkpoints ( + checkpoint_id INTEGER PRIMARY KEY, + position INTEGER + )", + "CREATE TABLE sapling_tree_shards ( + shard_index INTEGER PRIMARY KEY, + subtree_end_height INTEGER, + root_hash BLOB, + shard_data BLOB, + contains_marked INTEGER, + CONSTRAINT root_unique UNIQUE (root_hash) + )", "CREATE TABLE sapling_witnesses ( id_witness INTEGER PRIMARY KEY, note INTEGER NOT NULL, diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index b9e4a6bf04..f46d63ec39 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -2,14 +2,25 @@ //! tree data using the `shardtree` crate, and migrates existing witness data into these data //! structures. -use std::collections::HashSet; +use std::collections::{BTreeSet, HashSet}; -use rusqlite; +use incrementalmerkletree::Retention; +use rusqlite::{self, named_params, params}; use schemer; use schemer_rusqlite::RusqliteMigration; +use shardtree::ShardTree; use uuid::Uuid; -use crate::wallet::init::{migrations::received_notes_nullable_nf, WalletMigrationError}; +use zcash_primitives::{ + consensus::BlockHeight, + merkle_tree::{read_commitment_tree, read_incremental_witness}, + sapling, +}; + +use crate::wallet::{ + init::{migrations::received_notes_nullable_nf, WalletMigrationError}, + sapling::commitment_tree::WalletDbSaplingShardStore, +}; pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( 0x7da6489d, @@ -47,10 +58,129 @@ impl RusqliteMigration for Migration { ALTER TABLE sapling_received_notes ADD COLUMN commitment_tree_position INTEGER;", )?; + // Add shard persistence + transaction.execute_batch( + "CREATE TABLE sapling_tree_shards ( + shard_index INTEGER PRIMARY KEY, + subtree_end_height INTEGER, + root_hash BLOB, + shard_data BLOB, + contains_marked INTEGER NOT NULL, + CONSTRAINT root_unique UNIQUE (root_hash) + ); + CREATE TABLE sapling_tree_cap ( + cap_data BLOB NOT NULL + );", + )?; + + // Add checkpoint persistence + transaction.execute_batch( + "CREATE TABLE sapling_tree_checkpoints ( + checkpoint_id INTEGER PRIMARY KEY, + position INTEGER + ); + CREATE TABLE sapling_tree_checkpoint_marks_removed ( + checkpoint_id INTEGER NOT NULL, + mark_removed_position INTEGER NOT NULL, + FOREIGN KEY (checkpoint_id) REFERENCES sapling_tree_checkpoints(checkpoint_id) + );", + )?; + + let shard_store = WalletDbSaplingShardStore::from_connection(transaction)?; + let mut shard_tree: ShardTree< + _, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + { sapling::NOTE_COMMITMENT_TREE_DEPTH / 2 }, + > = ShardTree::new(shard_store, 100); + // Insert all the tree information that we can get from block-end commitment trees + { + let mut stmt_blocks = transaction.prepare("SELECT height, sapling_tree FROM blocks")?; + let mut stmt_update_block_sapling_tree_size = transaction + .prepare("UPDATE blocks SET sapling_commitment_tree_size = ? WHERE height = ?")?; + + let mut block_rows = stmt_blocks.query([])?; + while let Some(row) = block_rows.next()? { + let block_height: u32 = row.get(0)?; + let row_data: Vec = row.get(1)?; + let block_end_tree = read_commitment_tree::< + sapling::Node, + _, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + >(&row_data[..]) + .map_err(|e| { + rusqlite::Error::FromSqlConversionFailure( + row_data.len(), + rusqlite::types::Type::Blob, + Box::new(e), + ) + })?; + stmt_update_block_sapling_tree_size + .execute(params![block_end_tree.size(), block_height])?; + + if let Some(nonempty_frontier) = block_end_tree.to_frontier().value() { + shard_tree.insert_frontier_nodes( + nonempty_frontier.clone(), + Retention::Checkpoint { + id: BlockHeight::from(block_height), + is_marked: false, + }, + )?; + } + } + } + + // Insert all the tree information that we can get from existing incremental witnesses + { + let mut stmt_blocks = + transaction.prepare("SELECT note, block, witness FROM sapling_witnesses")?; + let mut stmt_set_note_position = transaction.prepare( + "UPDATE sapling_received_notes + SET commitment_tree_position = :position + WHERE id_note = :note_id", + )?; + let mut updated_note_positions = BTreeSet::new(); + let mut rows = stmt_blocks.query([])?; + while let Some(row) = rows.next()? { + let note_id: i64 = row.get(0)?; + let block_height: u32 = row.get(1)?; + let row_data: Vec = row.get(2)?; + let witness = read_incremental_witness::< + sapling::Node, + _, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + >(&row_data[..]) + .map_err(|e| { + rusqlite::Error::FromSqlConversionFailure( + row_data.len(), + rusqlite::types::Type::Blob, + Box::new(e), + ) + })?; + + let witnessed_position = witness.witnessed_position(); + if !updated_note_positions.contains(&witnessed_position) { + stmt_set_note_position.execute(named_params![ + ":note_id": note_id, + ":position": u64::from(witnessed_position) + ])?; + updated_note_positions.insert(witnessed_position); + } + + shard_tree.insert_witness_nodes(witness, BlockHeight::from(block_height))?; + } + } + Ok(()) } - fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + fn down(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + transaction.execute_batch( + "DROP TABLE sapling_tree_checkpoint_marks_removed; + DROP TABLE sapling_tree_checkpoints; + DROP TABLE sapling_tree_cap; + DROP TABLE sapling_tree_shards;", + )?; + Ok(()) } } diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs index a02ce4cd8d..ee7438ad64 100644 --- a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -1,9 +1,13 @@ +use either::Either; use incrementalmerkletree::Address; -use rusqlite; +use rusqlite::{self, named_params, OptionalExtension}; use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore}; +use std::io::{self, Cursor}; use zcash_primitives::{consensus::BlockHeight, sapling}; +use crate::serialization::read_shard; + pub struct WalletDbSaplingShardStore<'conn, 'a> { pub(crate) conn: &'a rusqlite::Transaction<'conn>, } @@ -19,14 +23,13 @@ impl<'conn, 'a> WalletDbSaplingShardStore<'conn, 'a> { impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { type H = sapling::Node; type CheckpointId = BlockHeight; - type Error = rusqlite::Error; + type Error = Either; fn get_shard( &self, - _shard_root: Address, + shard_root: Address, ) -> Result>, Self::Error> { - // SELECT shard_data FROM sapling_tree WHERE shard_index = shard_root.index - todo!() + get_shard(self.conn, shard_root) } fn last_shard(&self) -> Result>, Self::Error> { @@ -121,3 +124,23 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { todo!() } } + +pub(crate) fn get_shard( + conn: &rusqlite::Connection, + shard_root: Address, +) -> Result>, Either> { + conn.query_row( + "SELECT shard_data + FROM sapling_tree_shards + WHERE shard_index = :shard_index", + named_params![":shard_index": shard_root.index()], + |row| row.get::<_, Vec>(0), + ) + .optional() + .map_err(Either::Right)? + .map(|shard_data| { + let shard_tree = read_shard(&mut Cursor::new(shard_data)).map_err(Either::Left)?; + Ok(LocatedPrunableTree::from_parts(shard_root, shard_tree)) + }) + .transpose() +} From ade882d01cb74062d7cb70671e20c67575d04e02 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 2 Jun 2023 07:53:26 -0600 Subject: [PATCH 0025/1122] zcash_client_sqlite: Add shard & checkpoint insertion. --- zcash_client_sqlite/src/wallet.rs | 2 +- zcash_client_sqlite/src/wallet/init.rs | 4 +- .../init/migrations/add_transaction_views.rs | 4 +- .../migrations/received_notes_nullable_nf.rs | 2 +- .../init/migrations/shardtree_support.rs | 13 ++- .../init/migrations/v_transactions_net.rs | 8 +- .../src/wallet/sapling/commitment_tree.rs | 88 +++++++++++++++++-- 7 files changed, 98 insertions(+), 23 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 406db08c76..8f071d55d7 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -549,7 +549,7 @@ pub(crate) fn fully_scanned_height( |row| { let max_height: u32 = row.get(0)?; let sapling_tree_size: Option = row.get(1)?; - let sapling_tree: Vec = row.get(0)?; + let sapling_tree: Vec = row.get(2)?; Ok(( BlockHeight::from(max_height), sapling_tree_size, diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index e3485c8ab8..6fba9396c4 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -875,7 +875,7 @@ mod tests { // add a sapling sent note wdb.conn.execute( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, '')", + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00')", [], )?; @@ -1039,7 +1039,7 @@ mod tests { RecipientAddress::Transparent(*ufvk.default_address().0.transparent().unwrap()) .encode(&tests::network()); wdb.conn.execute( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, '')", + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00')", [], )?; wdb.conn.execute( diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs index 70694f8425..06efb13277 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs @@ -327,7 +327,7 @@ mod tests { .unwrap(); db_data.conn.execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, ''); + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00'); INSERT INTO transactions (block, id_tx, txid) VALUES (0, 0, ''); INSERT INTO sent_notes (tx, output_pool, output_index, from_account, address, value) @@ -460,7 +460,7 @@ mod tests { db_data .conn .execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, '');", + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00');", ) .unwrap(); db_data.conn.execute( diff --git a/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs b/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs index 811a1a0e5e..5567d60dc9 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs @@ -262,7 +262,7 @@ mod tests { // Tx 0 contains two received notes of 2 and 5 zatoshis that are controlled by account 0. db_data.conn.execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, ''); + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00'); INSERT INTO transactions (block, id_tx, txid) VALUES (0, 0, 'tx0'); INSERT INTO received_notes (tx, output_index, account, diversifier, value, rcm, nf, is_change) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index f46d63ec39..e16c36c8c3 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -65,7 +65,7 @@ impl RusqliteMigration for Migration { subtree_end_height INTEGER, root_hash BLOB, shard_data BLOB, - contains_marked INTEGER NOT NULL, + contains_marked INTEGER, CONSTRAINT root_unique UNIQUE (root_hash) ); CREATE TABLE sapling_tree_cap ( @@ -101,19 +101,24 @@ impl RusqliteMigration for Migration { let mut block_rows = stmt_blocks.query([])?; while let Some(row) = block_rows.next()? { let block_height: u32 = row.get(0)?; - let row_data: Vec = row.get(1)?; + let sapling_tree_data: Vec = row.get(1)?; + if sapling_tree_data == vec![0x00] { + continue; + } + let block_end_tree = read_commitment_tree::< sapling::Node, _, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, - >(&row_data[..]) + >(&sapling_tree_data[..]) .map_err(|e| { rusqlite::Error::FromSqlConversionFailure( - row_data.len(), + sapling_tree_data.len(), rusqlite::types::Type::Blob, Box::new(e), ) })?; + stmt_update_block_sapling_tree_size .execute(params![block_end_tree.size(), block_height])?; diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs index 10c3a26a9b..fc3ab73786 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs @@ -253,7 +253,7 @@ mod tests { // - Tx 0 contains two received notes of 2 and 5 zatoshis that are controlled by account 0. db_data.conn.execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, ''); + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00'); INSERT INTO transactions (block, id_tx, txid) VALUES (0, 0, 'tx0'); INSERT INTO received_notes (tx, output_index, account, diversifier, value, rcm, nf, is_change) @@ -265,7 +265,7 @@ mod tests { // of 2 zatoshis. This is representative of a historic transaction where no `sent_notes` // entry was created for the change value. db_data.conn.execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (1, 1, 1, ''); + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (1, 1, 1, x'00'); INSERT INTO transactions (block, id_tx, txid) VALUES (1, 1, 'tx1'); UPDATE received_notes SET spent = 1 WHERE tx = 0; INSERT INTO sent_notes (tx, output_pool, output_index, from_account, to_account, to_address, value) @@ -279,7 +279,7 @@ mod tests { // other half to the sending account as change. Also there's a random transparent utxo, // received, who knows where it came from but it's for account 0. db_data.conn.execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (2, 2, 2, ''); + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (2, 2, 2, x'00'); INSERT INTO transactions (block, id_tx, txid) VALUES (2, 2, 'tx2'); UPDATE received_notes SET spent = 2 WHERE tx = 1; INSERT INTO utxos (received_by_account, address, prevout_txid, prevout_idx, script, value_zat, height) @@ -297,7 +297,7 @@ mod tests { // - Tx 3 just receives transparent funds and does nothing else. For this to work, the // transaction must be retrieved by the wallet. db_data.conn.execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (3, 3, 3, ''); + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (3, 3, 3, x'00'); INSERT INTO transactions (block, id_tx, txid) VALUES (3, 3, 'tx3'); INSERT INTO utxos (received_by_account, address, prevout_txid, prevout_idx, script, value_zat, height) diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs index ee7438ad64..80c800fe6e 100644 --- a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -1,12 +1,14 @@ use either::Either; + use incrementalmerkletree::Address; use rusqlite::{self, named_params, OptionalExtension}; use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore}; + use std::io::{self, Cursor}; -use zcash_primitives::{consensus::BlockHeight, sapling}; +use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer, sapling}; -use crate::serialization::read_shard; +use crate::serialization::{read_shard, write_shard_v1}; pub struct WalletDbSaplingShardStore<'conn, 'a> { pub(crate) conn: &'a rusqlite::Transaction<'conn>, @@ -37,8 +39,8 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { todo!() } - fn put_shard(&mut self, _subtree: LocatedPrunableTree) -> Result<(), Self::Error> { - todo!() + fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { + put_shard(self.conn, subtree) } fn get_shard_roots(&self) -> Result, Self::Error> { @@ -68,14 +70,14 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { fn add_checkpoint( &mut self, - _checkpoint_id: Self::CheckpointId, - _checkpoint: Checkpoint, + checkpoint_id: Self::CheckpointId, + checkpoint: Checkpoint, ) -> Result<(), Self::Error> { - todo!() + add_checkpoint(self.conn, checkpoint_id, checkpoint) } fn checkpoint_count(&self) -> Result { - todo!() + checkpoint_count(self.conn) } fn get_checkpoint_at_depth( @@ -125,10 +127,12 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } } +type Error = Either; + pub(crate) fn get_shard( conn: &rusqlite::Connection, shard_root: Address, -) -> Result>, Either> { +) -> Result>, Error> { conn.query_row( "SELECT shard_data FROM sapling_tree_shards @@ -144,3 +148,69 @@ pub(crate) fn get_shard( }) .transpose() } + +pub(crate) fn put_shard( + conn: &rusqlite::Connection, + subtree: LocatedPrunableTree, +) -> Result<(), Error> { + let subtree_root_hash = subtree + .root() + .annotation() + .and_then(|ann| { + ann.as_ref().map(|rc| { + let mut root_hash = vec![]; + rc.write(&mut root_hash)?; + Ok(root_hash) + }) + }) + .transpose() + .map_err(Either::Left)?; + + let mut subtree_data = vec![]; + write_shard_v1(&mut subtree_data, subtree.root()).map_err(Either::Left)?; + + conn.prepare_cached( + "INSERT INTO sapling_tree_shards (shard_index, root_hash, shard_data) + VALUES (:shard_index, :root_hash, :shard_data) + ON CONFLICT (shard_index) DO UPDATE + SET root_hash = :root_hash, + shard_data = :shard_data", + ) + .and_then(|mut stmt_put_shard| { + stmt_put_shard.execute(named_params![ + ":shard_index": subtree.root_addr().index(), + ":root_hash": subtree_root_hash, + ":shard_data": subtree_data + ]) + }) + .map_err(Either::Right)?; + + Ok(()) +} + +pub(crate) fn add_checkpoint( + conn: &rusqlite::Transaction<'_>, + checkpoint_id: BlockHeight, + checkpoint: Checkpoint, +) -> Result<(), Error> { + conn.prepare_cached( + "INSERT INTO sapling_tree_checkpoints (checkpoint_id, position) + VALUES (:checkpoint_id, :position)", + ) + .and_then(|mut stmt_insert_checkpoint| { + stmt_insert_checkpoint.execute(named_params![ + ":checkpoint_id": u32::from(checkpoint_id), + ":position": checkpoint.position().map(u64::from) + ]) + }) + .map_err(Either::Right)?; + + Ok(()) +} + +pub(crate) fn checkpoint_count(conn: &rusqlite::Connection) -> Result { + conn.query_row("SELECT COUNT(*) FROM sapling_tree_checkpoints", [], |row| { + row.get::<_, usize>(0) + }) + .map_err(Either::Right) +} From d11f3d2acc11d92f3f819e8f765516304501f9ab Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 13 Jun 2023 11:20:18 -0600 Subject: [PATCH 0026/1122] zcash_client_sqlite: Add shardtree truncation & checkpoint operations. --- zcash_client_sqlite/src/chain.rs | 4 +- zcash_client_sqlite/src/lib.rs | 53 ++--- zcash_client_sqlite/src/wallet.rs | 17 +- zcash_client_sqlite/src/wallet/init.rs | 5 +- .../init/migrations/shardtree_support.rs | 3 + .../src/wallet/sapling/commitment_tree.rs | 181 +++++++++++++++--- 6 files changed, 209 insertions(+), 54 deletions(-) diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 11e065f9e4..d115482e20 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -539,7 +539,7 @@ mod tests { // "Rewind" to height of last scanned block db_data .transactionally(|wdb| { - truncate_to_height(&wdb.conn.0, &wdb.params, sapling_activation_height() + 1) + truncate_to_height(wdb.conn.0, &wdb.params, sapling_activation_height() + 1) }) .unwrap(); @@ -552,7 +552,7 @@ mod tests { // Rewind so that one block is dropped db_data .transactionally(|wdb| { - truncate_to_height(&wdb.conn.0, &wdb.params, sapling_activation_height()) + truncate_to_height(wdb.conn.0, &wdb.params, sapling_activation_height()) }) .unwrap(); diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 88b884b12f..cae3685e70 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -116,11 +116,11 @@ pub struct WalletDb { } /// A wrapper for a SQLite transaction affecting the wallet database. -pub struct SqlTransaction<'conn>(pub(crate) rusqlite::Transaction<'conn>); +pub struct SqlTransaction<'conn>(pub(crate) &'conn rusqlite::Transaction<'conn>); impl Borrow for SqlTransaction<'_> { fn borrow(&self) -> &rusqlite::Connection { - &self.0 + self.0 } } @@ -137,12 +137,13 @@ impl WalletDb { where F: FnOnce(&mut WalletDb, P>) -> Result, { + let tx = self.conn.transaction()?; let mut wdb = WalletDb { - conn: SqlTransaction(self.conn.transaction()?), + conn: SqlTransaction(&tx), params: self.params.clone(), }; let result = f(&mut wdb)?; - wdb.conn.0.commit()?; + tx.commit()?; Ok(result) } } @@ -334,7 +335,7 @@ impl WalletWrite for WalletDb seed: &SecretVec, ) -> Result<(AccountId, UnifiedSpendingKey), Self::Error> { self.transactionally(|wdb| { - let account = wallet::get_max_account_id(&wdb.conn.0)? + let account = wallet::get_max_account_id(wdb.conn.0)? .map(|a| AccountId::from(u32::from(a) + 1)) .unwrap_or_else(|| AccountId::from(0)); @@ -346,7 +347,7 @@ impl WalletWrite for WalletDb .map_err(|_| SqliteClientError::KeyDerivationError(account))?; let ufvk = usk.to_unified_full_viewing_key(); - wallet::add_account(&wdb.conn.0, &wdb.params, account, &ufvk)?; + wallet::add_account(wdb.conn.0, &wdb.params, account, &ufvk)?; Ok((account, usk)) }) @@ -360,7 +361,7 @@ impl WalletWrite for WalletDb |wdb| match wdb.get_unified_full_viewing_keys()?.get(&account) { Some(ufvk) => { let search_from = - match wallet::get_current_address(&wdb.conn.0, &wdb.params, account)? { + match wallet::get_current_address(wdb.conn.0, &wdb.params, account)? { Some((_, mut last_diversifier_index)) => { last_diversifier_index .increment() @@ -375,7 +376,7 @@ impl WalletWrite for WalletDb .ok_or(SqliteClientError::DiversifierIndexOutOfRange)?; wallet::insert_address( - &wdb.conn.0, + wdb.conn.0, &wdb.params, account, diversifier_index, @@ -399,7 +400,7 @@ impl WalletWrite for WalletDb // Insert the block into the database. let block_height = block.block_height; wallet::insert_block( - &wdb.conn.0, + wdb.conn.0, block_height, block.block_hash, block.block_time, @@ -408,16 +409,16 @@ impl WalletWrite for WalletDb let mut wallet_note_ids = vec![]; for tx in &block.transactions { - let tx_row = wallet::put_tx_meta(&wdb.conn.0, tx, block.block_height)?; + let tx_row = wallet::put_tx_meta(wdb.conn.0, tx, block.block_height)?; // Mark notes as spent and remove them from the scanning cache for spend in &tx.sapling_spends { - wallet::sapling::mark_sapling_note_spent(&wdb.conn.0, tx_row, spend.nf())?; + wallet::sapling::mark_sapling_note_spent(wdb.conn.0, tx_row, spend.nf())?; } for output in &tx.sapling_outputs { let received_note_id = - wallet::sapling::put_received_note(&wdb.conn.0, output, tx_row)?; + wallet::sapling::put_received_note(wdb.conn.0, output, tx_row)?; // Save witness for note. wallet_note_ids.push(received_note_id); @@ -435,7 +436,7 @@ impl WalletWrite for WalletDb })?; // Update now-expired transactions that didn't get mined. - wallet::update_expired_notes(&wdb.conn.0, block_height)?; + wallet::update_expired_notes(wdb.conn.0, block_height)?; Ok(wallet_note_ids) }) @@ -446,7 +447,7 @@ impl WalletWrite for WalletDb d_tx: DecryptedTransaction, ) -> Result { self.transactionally(|wdb| { - let tx_ref = wallet::put_tx_data(&wdb.conn.0, d_tx.tx, None, None)?; + let tx_ref = wallet::put_tx_data(wdb.conn.0, d_tx.tx, None, None)?; let mut spending_account_id: Option = None; for output in d_tx.sapling_outputs { @@ -459,7 +460,7 @@ impl WalletWrite for WalletDb }; wallet::put_sent_output( - &wdb.conn.0, + wdb.conn.0, &wdb.params, output.account, tx_ref, @@ -474,7 +475,7 @@ impl WalletWrite for WalletDb )?; if matches!(recipient, Recipient::InternalAccount(_, _)) { - wallet::sapling::put_received_note(&wdb.conn.0, output, tx_ref)?; + wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; } } TransferType::Incoming => { @@ -489,14 +490,14 @@ impl WalletWrite for WalletDb } } - wallet::sapling::put_received_note(&wdb.conn.0, output, tx_ref)?; + wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; } } // If any of the utxos spent in the transaction are ours, mark them as spent. #[cfg(feature = "transparent-inputs")] for txin in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vin.iter()) { - wallet::mark_transparent_utxo_spent(&wdb.conn.0, tx_ref, &txin.prevout)?; + wallet::mark_transparent_utxo_spent(wdb.conn.0, tx_ref, &txin.prevout)?; } // If we have some transparent outputs: @@ -513,7 +514,7 @@ impl WalletWrite for WalletDb for (output_index, txout) in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vout.iter()).enumerate() { if let Some(address) = txout.recipient_address() { wallet::put_sent_output( - &wdb.conn.0, + wdb.conn.0, &wdb.params, *account_id, tx_ref, @@ -534,7 +535,7 @@ impl WalletWrite for WalletDb fn store_sent_tx(&mut self, sent_tx: &SentTransaction) -> Result { self.transactionally(|wdb| { let tx_ref = wallet::put_tx_data( - &wdb.conn.0, + wdb.conn.0, sent_tx.tx, Some(sent_tx.fee_amount), Some(sent_tx.created), @@ -551,7 +552,7 @@ impl WalletWrite for WalletDb if let Some(bundle) = sent_tx.tx.sapling_bundle() { for spend in bundle.shielded_spends() { wallet::sapling::mark_sapling_note_spent( - &wdb.conn.0, + wdb.conn.0, tx_ref, spend.nullifier(), )?; @@ -560,12 +561,12 @@ impl WalletWrite for WalletDb #[cfg(feature = "transparent-inputs")] for utxo_outpoint in &sent_tx.utxos_spent { - wallet::mark_transparent_utxo_spent(&wdb.conn.0, tx_ref, utxo_outpoint)?; + wallet::mark_transparent_utxo_spent(wdb.conn.0, tx_ref, utxo_outpoint)?; } for output in &sent_tx.outputs { wallet::insert_sent_output( - &wdb.conn.0, + wdb.conn.0, &wdb.params, tx_ref, sent_tx.account, @@ -574,7 +575,7 @@ impl WalletWrite for WalletDb if let Some((account, note)) = output.sapling_change_to() { wallet::sapling::put_received_note( - &wdb.conn.0, + wdb.conn.0, &DecryptedOutput { index: output.output_index(), note: note.clone(), @@ -596,7 +597,7 @@ impl WalletWrite for WalletDb fn truncate_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error> { self.transactionally(|wdb| { - wallet::truncate_to_height(&wdb.conn.0, &wdb.params, block_height) + wallet::truncate_to_height(wdb.conn.0, &wdb.params, block_height) }) } @@ -661,7 +662,7 @@ impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb>>, { let mut shardtree = ShardTree::new( - WalletDbSaplingShardStore::from_connection(&self.conn.0) + WalletDbSaplingShardStore::from_connection(self.conn.0) .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?, 100, ); diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 8f071d55d7..6b2d150721 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -89,7 +89,9 @@ use zcash_client_backend::{ wallet::WalletTx, }; -use crate::{error::SqliteClientError, PRUNING_HEIGHT}; +use crate::{ + error::SqliteClientError, SqlTransaction, WalletCommitmentTrees, WalletDb, PRUNING_HEIGHT, +}; #[cfg(feature = "transparent-inputs")] use { @@ -637,7 +639,7 @@ pub(crate) fn get_min_unspent_height( /// block, this function does nothing. /// /// This should only be executed inside a transactional context. -pub(crate) fn truncate_to_height( +pub(crate) fn truncate_to_height( conn: &rusqlite::Transaction, params: &P, block_height: BlockHeight, @@ -662,7 +664,16 @@ pub(crate) fn truncate_to_height( // nothing to do if we're deleting back down to the max height if block_height < last_scanned_height { - // Decrement witnesses. + // Truncate the note commitment trees + let mut wdb = WalletDb { + conn: SqlTransaction(conn), + params: params.clone(), + }; + wdb.with_sapling_tree_mut(|tree| { + tree.truncate_removing_checkpoint(&block_height).map(|_| ()) + })?; + + // Remove any legacy Sapling witnesses conn.execute( "DELETE FROM sapling_witnesses WHERE block > ?", [u32::from(block_height)], diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 6fba9396c4..a98806ee65 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -237,7 +237,7 @@ pub fn init_accounts_table( // Insert accounts atomically for (account, key) in keys.iter() { - wallet::add_account(&wdb.conn.0, &wdb.params, *account, key)?; + wallet::add_account(wdb.conn.0, &wdb.params, *account, key)?; } Ok(()) @@ -394,6 +394,9 @@ mod tests { CONSTRAINT tx_output UNIQUE (tx, output_index) )", "CREATE TABLE sapling_tree_cap ( + -- cap_id exists only to be able to take advantage of `ON CONFLICT` + -- upsert functionality; the table will only ever contain one row + cap_id INTEGER PRIMARY KEY, cap_data BLOB NOT NULL )", "CREATE TABLE sapling_tree_checkpoint_marks_removed ( diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index e16c36c8c3..f22b03c20f 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -69,6 +69,9 @@ impl RusqliteMigration for Migration { CONSTRAINT root_unique UNIQUE (root_hash) ); CREATE TABLE sapling_tree_cap ( + -- cap_id exists only to be able to take advantage of `ON CONFLICT` + -- upsert functionality; the table will only ever contain one row + cap_id INTEGER PRIMARY KEY, cap_data BLOB NOT NULL );", )?; diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs index 80c800fe6e..912831fcf8 100644 --- a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -1,10 +1,14 @@ use either::Either; -use incrementalmerkletree::Address; -use rusqlite::{self, named_params, OptionalExtension}; -use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore}; +use incrementalmerkletree::{Address, Position}; +use rusqlite::{self, named_params, Connection, OptionalExtension}; +use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore, TreeState}; -use std::io::{self, Cursor}; +use std::{ + collections::BTreeSet, + io::{self, Cursor}, + ops::Deref, +}; use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer, sapling}; @@ -48,16 +52,16 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { todo!() } - fn truncate(&mut self, _from: Address) -> Result<(), Self::Error> { - todo!() + fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { + truncate(self.conn, from) } fn get_cap(&self) -> Result, Self::Error> { - todo!() + get_cap(self.conn) } - fn put_cap(&mut self, _cap: PrunableTree) -> Result<(), Self::Error> { - todo!() + fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { + put_cap(self.conn, cap) } fn min_checkpoint_id(&self) -> Result, Self::Error> { @@ -89,9 +93,9 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { fn get_checkpoint( &self, - _checkpoint_id: &Self::CheckpointId, + checkpoint_id: &Self::CheckpointId, ) -> Result, Self::Error> { - todo!() + get_checkpoint(self.conn, *checkpoint_id) } fn with_checkpoints(&mut self, _limit: usize, _callback: F) -> Result<(), Self::Error> @@ -103,27 +107,24 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { fn update_checkpoint_with( &mut self, - _checkpoint_id: &Self::CheckpointId, - _update: F, + checkpoint_id: &Self::CheckpointId, + update: F, ) -> Result where F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, { - todo!() + update_checkpoint_with(self.conn, *checkpoint_id, update) } - fn remove_checkpoint( - &mut self, - _checkpoint_id: &Self::CheckpointId, - ) -> Result<(), Self::Error> { - todo!() + fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { + remove_checkpoint(self.conn, *checkpoint_id) } fn truncate_checkpoints( &mut self, - _checkpoint_id: &Self::CheckpointId, + checkpoint_id: &Self::CheckpointId, ) -> Result<(), Self::Error> { - todo!() + truncate_checkpoints(self.conn, *checkpoint_id) } } @@ -134,7 +135,7 @@ pub(crate) fn get_shard( shard_root: Address, ) -> Result>, Error> { conn.query_row( - "SELECT shard_data + "SELECT shard_data FROM sapling_tree_shards WHERE shard_index = :shard_index", named_params![":shard_index": shard_root.index()], @@ -188,6 +189,47 @@ pub(crate) fn put_shard( Ok(()) } +pub(crate) fn truncate(conn: &rusqlite::Transaction<'_>, from: Address) -> Result<(), Error> { + conn.execute( + "DELETE FROM sapling_tree_shards WHERE shard_index >= ?", + [from.index()], + ) + .map_err(Either::Right) + .map(|_| ()) +} + +pub(crate) fn get_cap(conn: &rusqlite::Connection) -> Result, Error> { + conn.query_row("SELECT cap_data FROM sapling_tree_cap", [], |row| { + row.get::<_, Vec>(0) + }) + .optional() + .map_err(Either::Right)? + .map_or_else( + || Ok(PrunableTree::empty()), + |cap_data| read_shard(&mut Cursor::new(cap_data)).map_err(Either::Left), + ) +} + +pub(crate) fn put_cap( + conn: &rusqlite::Transaction<'_>, + cap: PrunableTree, +) -> Result<(), Error> { + let mut stmt = conn + .prepare_cached( + "INSERT INTO sapling_tree_cap (cap_id, cap_data) + VALUES (0, :cap_data) + ON CONFLICT (cap_id) DO UPDATE + SET cap_data = :cap_data", + ) + .map_err(Either::Right)?; + + let mut cap_data = vec![]; + write_shard_v1(&mut cap_data, &cap).map_err(Either::Left)?; + stmt.execute([cap_data]).map_err(Either::Right)?; + + Ok(()) +} + pub(crate) fn add_checkpoint( conn: &rusqlite::Transaction<'_>, checkpoint_id: BlockHeight, @@ -214,3 +256,98 @@ pub(crate) fn checkpoint_count(conn: &rusqlite::Connection) -> Result>( + conn: &C, + checkpoint_id: BlockHeight, +) -> Result, Either> { + let checkpoint_position = conn + .query_row( + "SELECT position + FROM sapling_tree_checkpoints + WHERE checkpoint_id = ?", + [u32::from(checkpoint_id)], + |row| { + row.get::<_, Option>(0) + .map(|opt| opt.map(Position::from)) + }, + ) + .optional() + .map_err(Either::Right)?; + + let mut marks_removed = BTreeSet::new(); + let mut stmt = conn + .prepare_cached( + "SELECT mark_removed_position + FROM sapling_tree_checkpoint_marks_removed + WHERE checkpoint_id = ?", + ) + .map_err(Either::Right)?; + let mut mark_removed_rows = stmt + .query([u32::from(checkpoint_id)]) + .map_err(Either::Right)?; + + while let Some(row) = mark_removed_rows.next().map_err(Either::Right)? { + marks_removed.insert( + row.get::<_, u64>(0) + .map(Position::from) + .map_err(Either::Right)?, + ); + } + + Ok(checkpoint_position.map(|pos_opt| { + Checkpoint::from_parts( + pos_opt.map_or(TreeState::Empty, TreeState::AtPosition), + marks_removed, + ) + })) +} + +pub(crate) fn update_checkpoint_with( + conn: &rusqlite::Transaction<'_>, + checkpoint_id: BlockHeight, + update: F, +) -> Result +where + F: Fn(&mut Checkpoint) -> Result<(), Error>, +{ + if let Some(mut c) = get_checkpoint(conn, checkpoint_id)? { + update(&mut c)?; + remove_checkpoint(conn, checkpoint_id)?; + add_checkpoint(conn, checkpoint_id, c)?; + Ok(true) + } else { + Ok(false) + } +} + +pub(crate) fn remove_checkpoint( + conn: &rusqlite::Transaction<'_>, + checkpoint_id: BlockHeight, +) -> Result<(), Error> { + conn.execute( + "DELETE FROM sapling_tree_checkpoints WHERE checkpoint_id = ?", + [u32::from(checkpoint_id)], + ) + .map_err(Either::Right)?; + + Ok(()) +} + +pub(crate) fn truncate_checkpoints( + conn: &rusqlite::Transaction<'_>, + checkpoint_id: BlockHeight, +) -> Result<(), Error> { + conn.execute( + "DELETE FROM sapling_tree_checkpoints WHERE checkpoint_id >= ?", + [u32::from(checkpoint_id)], + ) + .map_err(Either::Right)?; + + conn.execute( + "DELETE FROM sapling_tree_checkpoint_marks_removed WHERE checkpoint_id >= ?", + [u32::from(checkpoint_id)], + ) + .map_err(Either::Right)?; + Ok(()) +} From c42cffeb1d915cfd11f49bf12c3e37cf91680315 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 14 Jun 2023 15:34:28 -0600 Subject: [PATCH 0027/1122] zcash_client_backend: Replace `WalletWrite::advance_by_block` with `WalletWrite::put_block` Also, add assertions to prevent attempting the creation of zero-conf shielded spends. --- zcash_client_backend/CHANGELOG.md | 11 ++++++++-- zcash_client_backend/src/data_api.rs | 4 ++-- zcash_client_backend/src/data_api/chain.rs | 16 +++++++------- zcash_client_backend/src/data_api/wallet.rs | 22 ++++++++++++++++--- .../src/data_api/wallet/input_selection.rs | 17 +++++++++++++- zcash_client_backend/src/wallet.rs | 1 + zcash_client_sqlite/src/lib.rs | 2 +- 7 files changed, 56 insertions(+), 17 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index c896b6d702..28ab398643 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -7,9 +7,11 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added -- `impl Eq for zcash_client_backend::address::RecipientAddress` -- `impl Eq for zcash_client_backend::zip321::{Payment, TransactionRequest}` +- `impl Eq for address::RecipientAddress` +- `impl Eq for zip321::{Payment, TransactionRequest}` - `data_api::NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` +- `WalletWrite::put_block` +- `impl Debug` for `{data_api::wallet::input_selection::Proposal, wallet::ReceivedSaplingNote} ### Changed - MSRV is now 1.65.0. @@ -21,9 +23,14 @@ and this library adheres to Rust's notion of - `WalletRead::get_nullifiers` has been renamed to `WalletRead::get_sapling_nullifiers` and its signature has changed; it now subsumes the removed `WalletRead::get_all_nullifiers`. - `wallet::SpendableNote` has been renamed to `wallet::ReceivedSaplingNote`. +- `data_api::chain::scan_cached_blocks` now takes a `from_height` argument that + permits the caller to control the starting position of the scan range. +- `WalletWrite::advance_by_block` has been replaced by `WalletWrite::put_block` + to reflect the semantic change that scanning is no longer a linear operation. ### Removed - `WalletRead::get_all_nullifiers` +- `WalletWrite::advance_by_block` ## [0.9.0] - 2023-04-28 ### Added diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 80ad24f550..eafa0e89c9 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -401,7 +401,7 @@ pub trait WalletWrite: WalletRead { /// along with the note commitments that were detected when scanning the block for transactions /// pertaining to this wallet. #[allow(clippy::type_complexity)] - fn advance_by_block( + fn put_block( &mut self, block: PrunedBlock, ) -> Result, Self::Error>; @@ -660,7 +660,7 @@ pub mod testing { } #[allow(clippy::type_complexity)] - fn advance_by_block( + fn put_block( &mut self, _block: PrunedBlock, ) -> Result, Self::Error> { diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index ce0eb2a81b..dbec6768d1 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -175,7 +175,7 @@ where // comparing against the `validate_from` hash. block_source.with_blocks::<_, Infallible, Infallible>( - validate_from.map(|(h, _)| h), + validate_from.map(|(h, _)| h + 1), limit, move |block| { if let Some((valid_height, valid_hash)) = validate_from { @@ -260,18 +260,20 @@ where ); // Start at either the provided height, or where we synced up to previously. - let (last_scanned_height, commitment_tree_meta) = from_height.map_or_else( + let (from_height, commitment_tree_meta) = from_height.map_or_else( || { data_db.fully_scanned_height().map_or_else( |e| Err(Error::Wallet(e)), - |next| Ok(next.map_or_else(|| (None, None), |(h, m)| (Some(h), Some(m)))), + |last_scanned| { + Ok(last_scanned.map_or_else(|| (None, None), |(h, m)| (Some(h + 1), Some(m)))) + }, ) }, |h| Ok((Some(h), None)), )?; block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - last_scanned_height, + from_height, limit, |block: CompactBlock| { add_block_to_runner(params, block, &mut batch_runner); @@ -282,7 +284,7 @@ where batch_runner.flush(); block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - last_scanned_height, + from_height, limit, |block: CompactBlock| { let pruned_block = scan_block_with_runner( @@ -308,9 +310,7 @@ where .map(|out| (out.account(), *out.nf())) })); - data_db - .advance_by_block(pruned_block) - .map_err(Error::Wallet)?; + data_db.put_block(pruned_block).map_err(Error::Wallet)?; Ok(()) }, diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index b0930d9665..d37a033295 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -119,7 +119,8 @@ where /// can allow the sender to view the resulting notes on the blockchain. /// * `min_confirmations`: The minimum number of confirmations that a previously /// received note must have in the blockchain in order to be considered for being -/// spent. A value of 10 confirmations is recommended. +/// spent. A value of 10 confirmations is recommended and 0-conf transactions are +/// not supported. /// /// # Examples /// @@ -318,6 +319,10 @@ where ParamsT: consensus::Parameters + Clone, InputsT: InputSelector, { + assert!( + min_confirmations > 0, + "zero-conf transactions are not supported" + ); let account = wallet_db .get_account_for_ufvk(&usk.to_unified_full_viewing_key()) .map_err(Error::DataSource)? @@ -372,6 +377,10 @@ where ParamsT: consensus::Parameters + Clone, InputsT: InputSelector, { + assert!( + min_confirmations > 0, + "zero-conf transactions are not supported" + ); input_selector .propose_transaction( params, @@ -409,6 +418,10 @@ where DbT::NoteRef: Copy + Eq + Ord, InputsT: InputSelector, { + assert!( + min_confirmations > 0, + "zero-conf transactions are not supported" + ); input_selector .propose_shielding( params, @@ -453,6 +466,10 @@ where ParamsT: consensus::Parameters + Clone, FeeRuleT: FeeRule, { + assert!( + min_confirmations > 0, + "zero-conf transactions are not supported" + ); let account = wallet_db .get_account_for_ufvk(&usk.to_unified_full_viewing_key()) .map_err(Error::DataSource)? @@ -495,8 +512,7 @@ where selected, usk.sapling(), &dfvk, - min_confirmations - .try_into() + usize::try_from(min_confirmations - 1) .expect("min_confirmations should never be anywhere close to usize::MAX"), )? .ok_or(Error::NoteMismatch(selected.note_id))?; diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 403497c0d5..5017ae3350 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -1,8 +1,8 @@ //! Types related to the process of selecting inputs to be spent given a transaction request. use core::marker::PhantomData; -use std::collections::BTreeSet; use std::fmt; +use std::{collections::BTreeSet, fmt::Debug}; use zcash_primitives::{ consensus::{self, BlockHeight}, @@ -124,6 +124,21 @@ impl Proposal { } } +impl Debug for Proposal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Proposal") + .field("transaction_request", &self.transaction_request) + .field("transparent_inputs", &self.transparent_inputs) + .field("sapling_inputs", &self.sapling_inputs.len()) + .field("balance", &self.balance) + //.field("fee_rule", &self.fee_rule) + .field("min_target_height", &self.min_target_height) + .field("min_anchor_height", &self.min_anchor_height) + .field("is_shielding", &self.is_shielding) + .finish() + } +} + /// A strategy for selecting transaction inputs and proposing transaction outputs. /// /// Proposals should include only economically useful inputs, as determined by `Self::FeeRule`; diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index c702cfa73a..ccfe5a75b0 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -175,6 +175,7 @@ impl WalletSaplingOutput { /// Information about a note that is tracked by the wallet that is available for spending, /// with sufficient information for use in note selection. +#[derive(Debug)] pub struct ReceivedSaplingNote { pub note_id: NoteRef, pub diversifier: sapling::Diversifier, diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index cae3685e70..810a17bf44 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -392,7 +392,7 @@ impl WalletWrite for WalletDb #[tracing::instrument(skip_all, fields(height = u32::from(block.block_height)))] #[allow(clippy::type_complexity)] - fn advance_by_block( + fn put_block( &mut self, block: PrunedBlock, ) -> Result, Self::Error> { From 425b5e01d7da676eb3a92f04882cdee68131968b Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 14 Jun 2023 16:49:16 -0600 Subject: [PATCH 0028/1122] zcash_client_sqlite: Support shardtree checkpoint functionality --- zcash_client_sqlite/src/chain.rs | 113 +++++++---- zcash_client_sqlite/src/lib.rs | 2 +- zcash_client_sqlite/src/wallet.rs | 48 +++-- .../init/migrations/shardtree_support.rs | 3 +- zcash_client_sqlite/src/wallet/sapling.rs | 30 +-- .../src/wallet/sapling/commitment_tree.rs | 179 ++++++++++++++---- 6 files changed, 270 insertions(+), 105 deletions(-) diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index d115482e20..f5300de90c 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -23,12 +23,12 @@ pub mod migrations; /// Implements a traversal of `limit` blocks of the block cache database. /// -/// Starting at the next block above `last_scanned_height`, the `with_row` callback is invoked with -/// each block retrieved from the backing store. If the `limit` value provided is `None`, all -/// blocks are traversed up to the maximum height. +/// Starting at `from_height`, the `with_row` callback is invoked with each block retrieved from +/// the backing store. If the `limit` value provided is `None`, all blocks are traversed up to the +/// maximum height. pub(crate) fn blockdb_with_blocks( block_source: &BlockDb, - last_scanned_height: Option, + from_height: Option, limit: Option, mut with_row: F, ) -> Result<(), Error> @@ -43,15 +43,15 @@ where let mut stmt_blocks = block_source .0 .prepare( - "SELECT height, data FROM compactblocks - WHERE height > ? + "SELECT height, data FROM compactblocks + WHERE height >= ? ORDER BY height ASC LIMIT ?", ) .map_err(to_chain_error)?; let mut rows = stmt_blocks .query(params![ - last_scanned_height.map_or(0u32, u32::from), + from_height.map_or(0u32, u32::from), limit.unwrap_or(u32::max_value()), ]) .map_err(to_chain_error)?; @@ -191,13 +191,13 @@ pub(crate) fn blockmetadb_find_block( /// Implements a traversal of `limit` blocks of the filesystem-backed /// block cache. /// -/// Starting at the next block height above `last_scanned_height`, the `with_row` callback is -/// invoked with each block retrieved from the backing store. If the `limit` value provided is -/// `None`, all blocks are traversed up to the maximum height for which metadata is available. +/// Starting at `from_height`, the `with_row` callback is invoked with each block retrieved from +/// the backing store. If the `limit` value provided is `None`, all blocks are traversed up to the +/// maximum height for which metadata is available. #[cfg(feature = "unstable")] pub(crate) fn fsblockdb_with_blocks( cache: &FsBlockDb, - last_scanned_height: Option, + from_height: Option, limit: Option, mut with_block: F, ) -> Result<(), Error> @@ -214,7 +214,7 @@ where .prepare( "SELECT height, blockhash, time, sapling_outputs_count, orchard_actions_count FROM compactblocks_meta - WHERE height > ? + WHERE height >= ? ORDER BY height ASC LIMIT ?", ) .map_err(to_chain_error)?; @@ -222,7 +222,7 @@ where let rows = stmt_blocks .query_map( params![ - last_scanned_height.map_or(0u32, u32::from), + from_height.map_or(0u32, u32::from), limit.unwrap_or(u32::max_value()), ], |row| { @@ -269,14 +269,22 @@ mod tests { use tempfile::NamedTempFile; use zcash_primitives::{ - block::BlockHash, transaction::components::Amount, zip32::ExtendedSpendingKey, + block::BlockHash, + transaction::{components::Amount, fees::zip317::FeeRule}, + zip32::ExtendedSpendingKey, }; - use zcash_client_backend::data_api::chain::{ - error::{Cause, Error}, - scan_cached_blocks, validate_chain, + use zcash_client_backend::{ + address::RecipientAddress, + data_api::{ + chain::{error::Error, scan_cached_blocks, validate_chain}, + wallet::{input_selection::GreedyInputSelector, spend}, + WalletRead, WalletWrite, + }, + fees::{zip317::SingleOutputChangeStrategy, DustOutputPolicy}, + wallet::OvkPolicy, + zip321::{Payment, TransactionRequest}, }; - use zcash_client_backend::data_api::WalletRead; use crate::{ chain::init::init_cache_database, @@ -573,7 +581,7 @@ mod tests { } #[test] - fn scan_cached_blocks_requires_sequential_blocks() { + fn scan_cached_blocks_allows_blocks_out_of_order() { let cache_file = NamedTempFile::new().unwrap(); let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); init_cache_database(&db_cache).unwrap(); @@ -583,7 +591,9 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); + let seed = Secret::new([0u8; 32].to_vec()); + let (_, usk) = db_data.create_account(&seed).unwrap(); + let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Create a block with height SAPLING_ACTIVATION_HEIGHT let value = Amount::from_u64(50000).unwrap(); @@ -602,7 +612,7 @@ mod tests { value ); - // We cannot scan a block of height SAPLING_ACTIVATION_HEIGHT + 2 next + // Create blocks to reach SAPLING_ACTIVATION_HEIGHT + 2 let (cb2, _) = fake_compact_block( sapling_activation_height() + 1, cb1.hash(), @@ -619,25 +629,62 @@ mod tests { value, 2, ); + + // Scan the later block first insert_into_cache(&db_cache, &cb3); - match scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None) { - Err(Error::Chain(e)) => { - assert_matches!( - e.cause(), - Cause::BlockHeightDiscontinuity(h) if *h - == sapling_activation_height() + 2 - ); - } - Ok(_) | Err(_) => panic!("Should have failed"), - } + assert_matches!( + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + Some(sapling_activation_height() + 2), + None + ), + Ok(_) + ); - // If we add a block of height SAPLING_ACTIVATION_HEIGHT + 1, we can now scan both + // If we add a block of height SAPLING_ACTIVATION_HEIGHT + 1, we can now scan that insert_into_cache(&db_cache, &cb2); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + Some(sapling_activation_height() + 1), + Some(1), + ) + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), Amount::from_u64(150_000).unwrap() ); + + // We can spend the received notes + let req = TransactionRequest::new(vec![Payment { + recipient_address: RecipientAddress::Shielded(dfvk.default_address().1), + amount: Amount::from_u64(110_000).unwrap(), + memo: None, + label: None, + message: None, + other_params: vec![], + }]) + .unwrap(); + let input_selector = GreedyInputSelector::new( + SingleOutputChangeStrategy::new(FeeRule::standard()), + DustOutputPolicy::default(), + ); + assert_matches!( + spend( + &mut db_data, + &tests::network(), + crate::wallet::sapling::tests::test_prover(), + &input_selector, + &usk, + req, + OvkPolicy::Sender, + 1, + ), + Ok(_) + ); } #[test] diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 810a17bf44..0d46a1ce09 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -399,7 +399,7 @@ impl WalletWrite for WalletDb self.transactionally(|wdb| { // Insert the block into the database. let block_height = block.block_height; - wallet::insert_block( + wallet::put_block( wdb.conn.0, block_height, block.block_hash, diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 6b2d150721..7ecbc080c7 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -64,7 +64,7 @@ //! wallet. //! - `memo` the shielded memo associated with the output, if any. -use rusqlite::{self, named_params, params, OptionalExtension, ToSql}; +use rusqlite::{self, named_params, OptionalExtension, ToSql}; use std::collections::HashMap; use std::convert::TryFrom; use std::io::Cursor; @@ -735,15 +735,18 @@ pub(crate) fn get_unspent_transparent_outputs( FROM utxos u LEFT OUTER JOIN transactions tx ON tx.id_tx = u.spent_in_tx - WHERE u.address = ? - AND u.height <= ? + WHERE u.address = :address + AND u.height <= :max_height AND tx.block IS NULL", )?; let addr_str = address.encode(params); let mut utxos = Vec::::new(); - let mut rows = stmt_blocks.query(params![addr_str, u32::from(max_height)])?; + let mut rows = stmt_blocks.query(named_params![ + ":address": addr_str, + ":max_height": u32::from(max_height) + ])?; let excluded: BTreeSet = exclude.iter().cloned().collect(); while let Some(row) = rows.next()? { let txid: Vec = row.get(0)?; @@ -796,14 +799,17 @@ pub(crate) fn get_transparent_balances( FROM utxos u LEFT OUTER JOIN transactions tx ON tx.id_tx = u.spent_in_tx - WHERE u.received_by_account = ? - AND u.height <= ? + WHERE u.received_by_account = :account_id + AND u.height <= :max_height AND tx.block IS NULL GROUP BY u.address", )?; let mut res = HashMap::new(); - let mut rows = stmt_blocks.query(params![u32::from(account), u32::from(max_height)])?; + let mut rows = stmt_blocks.query(named_params![ + ":account_id": u32::from(account), + ":max_height": u32::from(max_height) + ])?; while let Some(row) = rows.next()? { let taddr_str: String = row.get(0)?; let taddr = TransparentAddress::decode(params, &taddr_str)?; @@ -816,14 +822,14 @@ pub(crate) fn get_transparent_balances( } /// Inserts information about a scanned block into the database. -pub(crate) fn insert_block( +pub(crate) fn put_block( conn: &rusqlite::Connection, block_height: BlockHeight, block_hash: BlockHash, block_time: u32, sapling_commitment_tree_size: Option, ) -> Result<(), SqliteClientError> { - let mut stmt_insert_block = conn.prepare_cached( + let mut stmt_upsert_block = conn.prepare_cached( "INSERT INTO blocks ( height, hash, @@ -831,14 +837,24 @@ pub(crate) fn insert_block( sapling_commitment_tree_size, sapling_tree ) - VALUES (?, ?, ?, ?, x'00')", + VALUES ( + :height, + :hash, + :block_time, + :sapling_commitment_tree_size, + x'00' + ) + ON CONFLICT (height) DO UPDATE + SET hash = :hash, + time = :block_time, + sapling_commitment_tree_size = :sapling_commitment_tree_size", )?; - stmt_insert_block.execute(params![ - u32::from(block_height), - &block_hash.0[..], - block_time, - sapling_commitment_tree_size + stmt_upsert_block.execute(named_params![ + ":height": u32::from(block_height), + ":hash": &block_hash.0[..], + ":block_time": block_time, + ":sapling_commitment_tree_size": sapling_commitment_tree_size ])?; Ok(()) @@ -981,7 +997,7 @@ pub(crate) fn put_legacy_transparent_utxo( #[cfg(feature = "transparent-inputs")] let mut stmt_upsert_legacy_transparent_utxo = conn.prepare_cached( "INSERT INTO utxos ( - prevout_txid, prevout_idx, + prevout_txid, prevout_idx, received_by_account, address, script, value_zat, height) VALUES diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index f22b03c20f..ded5b2d1f6 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -11,6 +11,7 @@ use schemer_rusqlite::RusqliteMigration; use shardtree::ShardTree; use uuid::Uuid; +use zcash_client_backend::data_api::SAPLING_SHARD_HEIGHT; use zcash_primitives::{ consensus::BlockHeight, merkle_tree::{read_commitment_tree, read_incremental_witness}, @@ -93,7 +94,7 @@ impl RusqliteMigration for Migration { let mut shard_tree: ShardTree< _, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, - { sapling::NOTE_COMMITMENT_TREE_DEPTH / 2 }, + SAPLING_SHARD_HEIGHT, > = ShardTree::new(shard_store, 100); // Insert all the tree information that we can get from block-end commitment trees { diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 511333ff7f..c763534d04 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -368,7 +368,7 @@ pub(crate) fn put_received_note( #[cfg(test)] #[allow(deprecated)] -mod tests { +pub(crate) mod tests { use rusqlite::Connection; use secrecy::Secret; use tempfile::NamedTempFile; @@ -427,7 +427,7 @@ mod tests { }, }; - fn test_prover() -> impl TxProver { + pub fn test_prover() -> impl TxProver { match LocalTxProver::with_default_location() { Some(tx_prover) => tx_prover, None => { @@ -463,7 +463,7 @@ mod tests { Amount::from_u64(1).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Err(data_api::error::Error::KeyNotRecognized) ); @@ -492,7 +492,7 @@ mod tests { Amount::from_u64(1).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Err(data_api::error::Error::ScanRequired) ); @@ -535,7 +535,7 @@ mod tests { Amount::from_u64(1).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Err(data_api::error::Error::InsufficientFunds { available, @@ -740,7 +740,7 @@ mod tests { Amount::from_u64(15000).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Ok(_) ); @@ -756,7 +756,7 @@ mod tests { Amount::from_u64(2000).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Err(data_api::error::Error::InsufficientFunds { available, @@ -791,7 +791,7 @@ mod tests { Amount::from_u64(2000).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Err(data_api::error::Error::InsufficientFunds { available, @@ -822,7 +822,7 @@ mod tests { Amount::from_u64(2000).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ) .unwrap(); } @@ -874,7 +874,7 @@ mod tests { Amount::from_u64(15000).unwrap(), None, ovk_policy, - 10, + 1, ) .unwrap(); @@ -962,7 +962,7 @@ mod tests { scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance - let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); + let (_, anchor_height) = db_data.get_target_and_anchor_heights(1).unwrap().unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -983,7 +983,7 @@ mod tests { Amount::from_u64(50000).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Ok(_) ); @@ -1039,7 +1039,7 @@ mod tests { Amount::from_u64(50000).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Ok(_) ); @@ -1193,7 +1193,7 @@ mod tests { DustOutputPolicy::default(), ); - // Add funds to the wallet + // Ensure that the wallet has at least one block let (cb, _) = fake_compact_block( sapling_activation_height(), BlockHash([0; 32]), @@ -1215,7 +1215,7 @@ mod tests { &usk, &[*taddr], &MemoBytes::empty(), - 0 + 1 ), Ok(_) ); diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs index 912831fcf8..5ea9e29fe8 100644 --- a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -1,19 +1,22 @@ use either::Either; - -use incrementalmerkletree::{Address, Position}; use rusqlite::{self, named_params, Connection, OptionalExtension}; -use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore, TreeState}; - use std::{ collections::BTreeSet, io::{self, Cursor}, ops::Deref, }; +use incrementalmerkletree::{Address, Level, Position}; +use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore, TreeState}; + use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer, sapling}; +use zcash_client_backend::data_api::SAPLING_SHARD_HEIGHT; + use crate::serialization::{read_shard, write_shard_v1}; +const SHARD_ROOT_LEVEL: Level = Level::new(SAPLING_SHARD_HEIGHT); + pub struct WalletDbSaplingShardStore<'conn, 'a> { pub(crate) conn: &'a rusqlite::Transaction<'conn>, } @@ -39,8 +42,7 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } fn last_shard(&self) -> Result>, Self::Error> { - // SELECT shard_data FROM sapling_tree ORDER BY shard_index DESC LIMIT 1 - todo!() + last_shard(self.conn) } fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { @@ -48,8 +50,7 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } fn get_shard_roots(&self) -> Result, Self::Error> { - // SELECT - todo!() + get_shard_roots(self.conn) } fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { @@ -86,9 +87,9 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { fn get_checkpoint_at_depth( &self, - _checkpoint_depth: usize, + checkpoint_depth: usize, ) -> Result, Self::Error> { - todo!() + get_checkpoint_at_depth(self.conn, checkpoint_depth) } fn get_checkpoint( @@ -150,6 +151,31 @@ pub(crate) fn get_shard( .transpose() } +pub(crate) fn last_shard( + conn: &rusqlite::Connection, +) -> Result>, Error> { + conn.query_row( + "SELECT shard_index, shard_data + FROM sapling_tree_shards + ORDER BY shard_index DESC + LIMIT 1", + [], + |row| { + let shard_index: u64 = row.get(0)?; + let shard_data: Vec = row.get(1)?; + Ok((shard_index, shard_data)) + }, + ) + .optional() + .map_err(Either::Right)? + .map(|(shard_index, shard_data)| { + let shard_root = Address::from_parts(SHARD_ROOT_LEVEL, shard_index); + let shard_tree = read_shard(&mut Cursor::new(shard_data)).map_err(Either::Left)?; + Ok(LocatedPrunableTree::from_parts(shard_root, shard_tree)) + }) + .transpose() +} + pub(crate) fn put_shard( conn: &rusqlite::Connection, subtree: LocatedPrunableTree, @@ -172,10 +198,10 @@ pub(crate) fn put_shard( conn.prepare_cached( "INSERT INTO sapling_tree_shards (shard_index, root_hash, shard_data) - VALUES (:shard_index, :root_hash, :shard_data) - ON CONFLICT (shard_index) DO UPDATE - SET root_hash = :root_hash, - shard_data = :shard_data", + VALUES (:shard_index, :root_hash, :shard_data) + ON CONFLICT (shard_index) DO UPDATE + SET root_hash = :root_hash, + shard_data = :shard_data", ) .and_then(|mut stmt_put_shard| { stmt_put_shard.execute(named_params![ @@ -189,6 +215,22 @@ pub(crate) fn put_shard( Ok(()) } +pub(crate) fn get_shard_roots(conn: &rusqlite::Connection) -> Result, Error> { + let mut stmt = conn + .prepare("SELECT shard_index FROM sapling_tree_shards ORDER BY shard_index") + .map_err(Either::Right)?; + let mut rows = stmt.query([]).map_err(Either::Right)?; + + let mut res = vec![]; + while let Some(row) = rows.next().map_err(Either::Right)? { + res.push(Address::from_parts( + SHARD_ROOT_LEVEL, + row.get(0).map_err(Either::Right)?, + )); + } + Ok(res) +} + pub(crate) fn truncate(conn: &rusqlite::Transaction<'_>, from: Address) -> Result<(), Error> { conn.execute( "DELETE FROM sapling_tree_shards WHERE shard_index >= ?", @@ -264,8 +306,8 @@ pub(crate) fn get_checkpoint>( let checkpoint_position = conn .query_row( "SELECT position - FROM sapling_tree_checkpoints - WHERE checkpoint_id = ?", + FROM sapling_tree_checkpoints + WHERE checkpoint_id = ?", [u32::from(checkpoint_id)], |row| { row.get::<_, Option>(0) @@ -275,32 +317,91 @@ pub(crate) fn get_checkpoint>( .optional() .map_err(Either::Right)?; - let mut marks_removed = BTreeSet::new(); - let mut stmt = conn - .prepare_cached( - "SELECT mark_removed_position - FROM sapling_tree_checkpoint_marks_removed - WHERE checkpoint_id = ?", + checkpoint_position + .map(|pos_opt| { + let mut marks_removed = BTreeSet::new(); + let mut stmt = conn + .prepare_cached( + "SELECT mark_removed_position + FROM sapling_tree_checkpoint_marks_removed + WHERE checkpoint_id = ?", + ) + .map_err(Either::Right)?; + let mut mark_removed_rows = stmt + .query([u32::from(checkpoint_id)]) + .map_err(Either::Right)?; + + while let Some(row) = mark_removed_rows.next().map_err(Either::Right)? { + marks_removed.insert( + row.get::<_, u64>(0) + .map(Position::from) + .map_err(Either::Right)?, + ); + } + + Ok(Checkpoint::from_parts( + pos_opt.map_or(TreeState::Empty, TreeState::AtPosition), + marks_removed, + )) + }) + .transpose() +} + +pub(crate) fn get_checkpoint_at_depth>( + conn: &C, + checkpoint_depth: usize, +) -> Result, Either> { + let checkpoint_parts = conn + .query_row( + "SELECT checkpoint_id, position + FROM sapling_tree_checkpoints + ORDER BY checkpoint_id DESC + LIMIT 1 + OFFSET :offset", + named_params![":offset": checkpoint_depth], + |row| { + let checkpoint_id: u32 = row.get(0)?; + let position: Option = row.get(1)?; + Ok(( + BlockHeight::from(checkpoint_id), + position.map(Position::from), + )) + }, ) - .map_err(Either::Right)?; - let mut mark_removed_rows = stmt - .query([u32::from(checkpoint_id)]) + .optional() .map_err(Either::Right)?; - while let Some(row) = mark_removed_rows.next().map_err(Either::Right)? { - marks_removed.insert( - row.get::<_, u64>(0) - .map(Position::from) - .map_err(Either::Right)?, - ); - } - - Ok(checkpoint_position.map(|pos_opt| { - Checkpoint::from_parts( - pos_opt.map_or(TreeState::Empty, TreeState::AtPosition), - marks_removed, - ) - })) + checkpoint_parts + .map(|(checkpoint_id, pos_opt)| { + let mut marks_removed = BTreeSet::new(); + let mut stmt = conn + .prepare_cached( + "SELECT mark_removed_position + FROM sapling_tree_checkpoint_marks_removed + WHERE checkpoint_id = ?", + ) + .map_err(Either::Right)?; + let mut mark_removed_rows = stmt + .query([u32::from(checkpoint_id)]) + .map_err(Either::Right)?; + + while let Some(row) = mark_removed_rows.next().map_err(Either::Right)? { + marks_removed.insert( + row.get::<_, u64>(0) + .map(Position::from) + .map_err(Either::Right)?, + ); + } + + Ok(( + checkpoint_id, + Checkpoint::from_parts( + pos_opt.map_or(TreeState::Empty, TreeState::AtPosition), + marks_removed, + ), + )) + }) + .transpose() } pub(crate) fn update_checkpoint_with( From 0a4236f725a3f46fce7247fa0860ba44871eb8d8 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 15 Jun 2023 13:50:07 -0600 Subject: [PATCH 0029/1122] zcash_client_sqlite: Add tests for sqlite-backed ShardTree & fix revealed issues. --- zcash_client_sqlite/src/lib.rs | 14 +- zcash_client_sqlite/src/wallet/init.rs | 1 + .../init/migrations/shardtree_support.rs | 8 +- .../src/wallet/sapling/commitment_tree.rs | 425 +++++++++++++++--- zcash_primitives/src/consensus.rs | 6 + zcash_primitives/src/merkle_tree.rs | 18 + 6 files changed, 395 insertions(+), 77 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 0d46a1ce09..9df48d5cac 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -66,9 +66,7 @@ use zcash_client_backend::{ DecryptedOutput, TransferType, }; -use crate::{ - error::SqliteClientError, wallet::sapling::commitment_tree::WalletDbSaplingShardStore, -}; +use crate::{error::SqliteClientError, wallet::sapling::commitment_tree::SqliteShardStore}; #[cfg(feature = "unstable")] use { @@ -617,7 +615,8 @@ impl WalletWrite for WalletDb impl WalletCommitmentTrees for WalletDb { type Error = Either; - type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; + type SaplingShardStore<'a> = + SqliteShardStore<&'a rusqlite::Transaction<'a>, sapling::Node, SAPLING_SHARD_HEIGHT>; fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result where @@ -634,7 +633,7 @@ impl WalletCommitmentTrees for WalletDb WalletCommitmentTrees for WalletDb WalletCommitmentTrees for WalletDb, P> { type Error = Either; - type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; + type SaplingShardStore<'a> = + SqliteShardStore<&'a rusqlite::Transaction<'a>, sapling::Node, SAPLING_SHARD_HEIGHT>; fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result where @@ -662,7 +662,7 @@ impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb>>, { let mut shardtree = ShardTree::new( - WalletDbSaplingShardStore::from_connection(self.conn.0) + SqliteShardStore::from_connection(self.conn.0) .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?, 100, ); diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index a98806ee65..c730aea735 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -403,6 +403,7 @@ mod tests { checkpoint_id INTEGER NOT NULL, mark_removed_position INTEGER NOT NULL, FOREIGN KEY (checkpoint_id) REFERENCES sapling_tree_checkpoints(checkpoint_id) + ON DELETE CASCADE )", "CREATE TABLE sapling_tree_checkpoints ( checkpoint_id INTEGER PRIMARY KEY, diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index ded5b2d1f6..6a597d56c7 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -20,7 +20,7 @@ use zcash_primitives::{ use crate::wallet::{ init::{migrations::received_notes_nullable_nf, WalletMigrationError}, - sapling::commitment_tree::WalletDbSaplingShardStore, + sapling::commitment_tree::SqliteShardStore, }; pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( @@ -87,10 +87,14 @@ impl RusqliteMigration for Migration { checkpoint_id INTEGER NOT NULL, mark_removed_position INTEGER NOT NULL, FOREIGN KEY (checkpoint_id) REFERENCES sapling_tree_checkpoints(checkpoint_id) + ON DELETE CASCADE );", )?; - let shard_store = WalletDbSaplingShardStore::from_connection(transaction)?; + let shard_store = + SqliteShardStore::<_, sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( + transaction, + )?; let mut shard_tree: ShardTree< _, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs index 5ea9e29fe8..f685350c14 100644 --- a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -1,36 +1,38 @@ use either::Either; -use rusqlite::{self, named_params, Connection, OptionalExtension}; +use rusqlite::{self, named_params, OptionalExtension}; use std::{ collections::BTreeSet, io::{self, Cursor}, - ops::Deref, + marker::PhantomData, }; use incrementalmerkletree::{Address, Level, Position}; use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore, TreeState}; -use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer, sapling}; - -use zcash_client_backend::data_api::SAPLING_SHARD_HEIGHT; +use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer}; use crate::serialization::{read_shard, write_shard_v1}; -const SHARD_ROOT_LEVEL: Level = Level::new(SAPLING_SHARD_HEIGHT); - -pub struct WalletDbSaplingShardStore<'conn, 'a> { - pub(crate) conn: &'a rusqlite::Transaction<'conn>, +pub struct SqliteShardStore { + pub(crate) conn: C, + _hash_type: PhantomData, } -impl<'conn, 'a> WalletDbSaplingShardStore<'conn, 'a> { - pub(crate) fn from_connection( - conn: &'a rusqlite::Transaction<'conn>, - ) -> Result { - Ok(WalletDbSaplingShardStore { conn }) +impl SqliteShardStore { + const SHARD_ROOT_LEVEL: Level = Level::new(SHARD_HEIGHT); + + pub(crate) fn from_connection(conn: C) -> Result { + Ok(SqliteShardStore { + conn, + _hash_type: PhantomData, + }) } } -impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { - type H = sapling::Node; +impl<'conn, 'a: 'conn, H: HashSer, const SHARD_HEIGHT: u8> ShardStore + for SqliteShardStore<&'a rusqlite::Transaction<'conn>, H, SHARD_HEIGHT> +{ + type H = H; type CheckpointId = BlockHeight; type Error = Either; @@ -42,7 +44,7 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } fn last_shard(&self) -> Result>, Self::Error> { - last_shard(self.conn) + last_shard(self.conn, Self::SHARD_ROOT_LEVEL) } fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { @@ -50,7 +52,7 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } fn get_shard_roots(&self) -> Result, Self::Error> { - get_shard_roots(self.conn) + get_shard_roots(self.conn, Self::SHARD_ROOT_LEVEL) } fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { @@ -66,11 +68,11 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } fn min_checkpoint_id(&self) -> Result, Self::Error> { - todo!() + min_checkpoint_id(self.conn) } fn max_checkpoint_id(&self) -> Result, Self::Error> { - todo!() + max_checkpoint_id(self.conn) } fn add_checkpoint( @@ -99,11 +101,11 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { get_checkpoint(self.conn, *checkpoint_id) } - fn with_checkpoints(&mut self, _limit: usize, _callback: F) -> Result<(), Self::Error> + fn with_checkpoints(&mut self, limit: usize, callback: F) -> Result<(), Self::Error> where F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, { - todo!() + with_checkpoints(self.conn, limit, callback) } fn update_checkpoint_with( @@ -129,12 +131,128 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } } +impl ShardStore + for SqliteShardStore +{ + type H = H; + type CheckpointId = BlockHeight; + type Error = Either; + + fn get_shard( + &self, + shard_root: Address, + ) -> Result>, Self::Error> { + get_shard(&self.conn, shard_root) + } + + fn last_shard(&self) -> Result>, Self::Error> { + last_shard(&self.conn, Self::SHARD_ROOT_LEVEL) + } + + fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { + let tx = self.conn.transaction().map_err(Either::Right)?; + put_shard(&tx, subtree)?; + tx.commit().map_err(Either::Right)?; + Ok(()) + } + + fn get_shard_roots(&self) -> Result, Self::Error> { + get_shard_roots(&self.conn, Self::SHARD_ROOT_LEVEL) + } + + fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { + truncate(&self.conn, from) + } + + fn get_cap(&self) -> Result, Self::Error> { + get_cap(&self.conn) + } + + fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { + put_cap(&self.conn, cap) + } + + fn min_checkpoint_id(&self) -> Result, Self::Error> { + min_checkpoint_id(&self.conn) + } + + fn max_checkpoint_id(&self) -> Result, Self::Error> { + max_checkpoint_id(&self.conn) + } + + fn add_checkpoint( + &mut self, + checkpoint_id: Self::CheckpointId, + checkpoint: Checkpoint, + ) -> Result<(), Self::Error> { + let tx = self.conn.transaction().map_err(Either::Right)?; + add_checkpoint(&tx, checkpoint_id, checkpoint)?; + tx.commit().map_err(Either::Right) + } + + fn checkpoint_count(&self) -> Result { + checkpoint_count(&self.conn) + } + + fn get_checkpoint_at_depth( + &self, + checkpoint_depth: usize, + ) -> Result, Self::Error> { + get_checkpoint_at_depth(&self.conn, checkpoint_depth) + } + + fn get_checkpoint( + &self, + checkpoint_id: &Self::CheckpointId, + ) -> Result, Self::Error> { + get_checkpoint(&self.conn, *checkpoint_id) + } + + fn with_checkpoints(&mut self, limit: usize, callback: F) -> Result<(), Self::Error> + where + F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, + { + let tx = self.conn.transaction().map_err(Either::Right)?; + with_checkpoints(&tx, limit, callback)?; + tx.commit().map_err(Either::Right) + } + + fn update_checkpoint_with( + &mut self, + checkpoint_id: &Self::CheckpointId, + update: F, + ) -> Result + where + F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, + { + let tx = self.conn.transaction().map_err(Either::Right)?; + let result = update_checkpoint_with(&tx, *checkpoint_id, update)?; + tx.commit().map_err(Either::Right)?; + Ok(result) + } + + fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { + let tx = self.conn.transaction().map_err(Either::Right)?; + remove_checkpoint(&tx, *checkpoint_id)?; + tx.commit().map_err(Either::Right) + } + + fn truncate_checkpoints( + &mut self, + checkpoint_id: &Self::CheckpointId, + ) -> Result<(), Self::Error> { + let tx = self.conn.transaction().map_err(Either::Right)?; + truncate_checkpoints(&tx, *checkpoint_id)?; + tx.commit().map_err(Either::Right) + } +} + type Error = Either; -pub(crate) fn get_shard( +pub(crate) fn get_shard( conn: &rusqlite::Connection, shard_root: Address, -) -> Result>, Error> { +) -> Result>, Error> { conn.query_row( "SELECT shard_data FROM sapling_tree_shards @@ -151,9 +269,10 @@ pub(crate) fn get_shard( .transpose() } -pub(crate) fn last_shard( +pub(crate) fn last_shard( conn: &rusqlite::Connection, -) -> Result>, Error> { + shard_root_level: Level, +) -> Result>, Error> { conn.query_row( "SELECT shard_index, shard_data FROM sapling_tree_shards @@ -169,16 +288,16 @@ pub(crate) fn last_shard( .optional() .map_err(Either::Right)? .map(|(shard_index, shard_data)| { - let shard_root = Address::from_parts(SHARD_ROOT_LEVEL, shard_index); + let shard_root = Address::from_parts(shard_root_level, shard_index); let shard_tree = read_shard(&mut Cursor::new(shard_data)).map_err(Either::Left)?; Ok(LocatedPrunableTree::from_parts(shard_root, shard_tree)) }) .transpose() } -pub(crate) fn put_shard( - conn: &rusqlite::Connection, - subtree: LocatedPrunableTree, +pub(crate) fn put_shard( + conn: &rusqlite::Transaction<'_>, + subtree: LocatedPrunableTree, ) -> Result<(), Error> { let subtree_root_hash = subtree .root() @@ -196,26 +315,31 @@ pub(crate) fn put_shard( let mut subtree_data = vec![]; write_shard_v1(&mut subtree_data, subtree.root()).map_err(Either::Left)?; - conn.prepare_cached( - "INSERT INTO sapling_tree_shards (shard_index, root_hash, shard_data) - VALUES (:shard_index, :root_hash, :shard_data) - ON CONFLICT (shard_index) DO UPDATE - SET root_hash = :root_hash, - shard_data = :shard_data", - ) - .and_then(|mut stmt_put_shard| { - stmt_put_shard.execute(named_params![ + let mut stmt_put_shard = conn + .prepare_cached( + "INSERT INTO sapling_tree_shards (shard_index, root_hash, shard_data) + VALUES (:shard_index, :root_hash, :shard_data) + ON CONFLICT (shard_index) DO UPDATE + SET root_hash = :root_hash, + shard_data = :shard_data", + ) + .map_err(Either::Right)?; + + stmt_put_shard + .execute(named_params![ ":shard_index": subtree.root_addr().index(), ":root_hash": subtree_root_hash, ":shard_data": subtree_data ]) - }) - .map_err(Either::Right)?; + .map_err(Either::Right)?; Ok(()) } -pub(crate) fn get_shard_roots(conn: &rusqlite::Connection) -> Result, Error> { +pub(crate) fn get_shard_roots( + conn: &rusqlite::Connection, + shard_root_level: Level, +) -> Result, Error> { let mut stmt = conn .prepare("SELECT shard_index FROM sapling_tree_shards ORDER BY shard_index") .map_err(Either::Right)?; @@ -224,14 +348,14 @@ pub(crate) fn get_shard_roots(conn: &rusqlite::Connection) -> Result, from: Address) -> Result<(), Error> { +pub(crate) fn truncate(conn: &rusqlite::Connection, from: Address) -> Result<(), Error> { conn.execute( "DELETE FROM sapling_tree_shards WHERE shard_index >= ?", [from.index()], @@ -240,7 +364,7 @@ pub(crate) fn truncate(conn: &rusqlite::Transaction<'_>, from: Address) -> Resul .map(|_| ()) } -pub(crate) fn get_cap(conn: &rusqlite::Connection) -> Result, Error> { +pub(crate) fn get_cap(conn: &rusqlite::Connection) -> Result, Error> { conn.query_row("SELECT cap_data FROM sapling_tree_cap", [], |row| { row.get::<_, Vec>(0) }) @@ -252,9 +376,9 @@ pub(crate) fn get_cap(conn: &rusqlite::Connection) -> Result, - cap: PrunableTree, +pub(crate) fn put_cap( + conn: &rusqlite::Connection, + cap: PrunableTree, ) -> Result<(), Error> { let mut stmt = conn .prepare_cached( @@ -272,22 +396,62 @@ pub(crate) fn put_cap( Ok(()) } +pub(crate) fn min_checkpoint_id(conn: &rusqlite::Connection) -> Result, Error> { + conn.query_row( + "SELECT MIN(checkpoint_id) FROM sapling_tree_checkpoints", + [], + |row| { + row.get::<_, Option>(0) + .map(|opt| opt.map(BlockHeight::from)) + }, + ) + .map_err(Either::Right) +} + +pub(crate) fn max_checkpoint_id(conn: &rusqlite::Connection) -> Result, Error> { + conn.query_row( + "SELECT MAX(checkpoint_id) FROM sapling_tree_checkpoints", + [], + |row| { + row.get::<_, Option>(0) + .map(|opt| opt.map(BlockHeight::from)) + }, + ) + .map_err(Either::Right) +} + pub(crate) fn add_checkpoint( conn: &rusqlite::Transaction<'_>, checkpoint_id: BlockHeight, checkpoint: Checkpoint, ) -> Result<(), Error> { - conn.prepare_cached( - "INSERT INTO sapling_tree_checkpoints (checkpoint_id, position) - VALUES (:checkpoint_id, :position)", - ) - .and_then(|mut stmt_insert_checkpoint| { - stmt_insert_checkpoint.execute(named_params![ + let mut stmt_insert_checkpoint = conn + .prepare_cached( + "INSERT INTO sapling_tree_checkpoints (checkpoint_id, position) + VALUES (:checkpoint_id, :position)", + ) + .map_err(Either::Right)?; + + stmt_insert_checkpoint + .execute(named_params![ ":checkpoint_id": u32::from(checkpoint_id), ":position": checkpoint.position().map(u64::from) ]) - }) - .map_err(Either::Right)?; + .map_err(Either::Right)?; + + let mut stmt_insert_mark_removed = conn.prepare_cached( + "INSERT INTO sapling_tree_checkpoint_marks_removed (checkpoint_id, mark_removed_position) + VALUES (:checkpoint_id, :position)", + ).map_err(Either::Right)?; + + for pos in checkpoint.marks_removed() { + stmt_insert_mark_removed + .execute(named_params![ + ":checkpoint_id": u32::from(checkpoint_id), + ":position": u64::from(*pos) + ]) + .map_err(Either::Right)?; + } Ok(()) } @@ -299,10 +463,10 @@ pub(crate) fn checkpoint_count(conn: &rusqlite::Connection) -> Result>( - conn: &C, +pub(crate) fn get_checkpoint( + conn: &rusqlite::Connection, checkpoint_id: BlockHeight, -) -> Result, Either> { +) -> Result, Error> { let checkpoint_position = conn .query_row( "SELECT position @@ -347,10 +511,14 @@ pub(crate) fn get_checkpoint>( .transpose() } -pub(crate) fn get_checkpoint_at_depth>( - conn: &C, +pub(crate) fn get_checkpoint_at_depth( + conn: &rusqlite::Connection, checkpoint_depth: usize, -) -> Result, Either> { +) -> Result, Error> { + if checkpoint_depth == 0 { + return Ok(None); + } + let checkpoint_parts = conn .query_row( "SELECT checkpoint_id, position @@ -358,7 +526,7 @@ pub(crate) fn get_checkpoint_at_depth>( ORDER BY checkpoint_id DESC LIMIT 1 OFFSET :offset", - named_params![":offset": checkpoint_depth], + named_params![":offset": checkpoint_depth - 1], |row| { let checkpoint_id: u32 = row.get(0)?; let position: Option = row.get(1)?; @@ -404,6 +572,62 @@ pub(crate) fn get_checkpoint_at_depth>( .transpose() } +pub(crate) fn with_checkpoints( + conn: &rusqlite::Transaction<'_>, + limit: usize, + mut callback: F, +) -> Result<(), Error> +where + F: FnMut(&BlockHeight, &Checkpoint) -> Result<(), Error>, +{ + let mut stmt_get_checkpoints = conn + .prepare_cached( + "SELECT checkpoint_id, position + FROM sapling_tree_checkpoints + LIMIT :limit", + ) + .map_err(Either::Right)?; + + let mut stmt_get_checkpoint_marks_removed = conn + .prepare_cached( + "SELECT mark_removed_position + FROM sapling_tree_checkpoint_marks_removed + WHERE checkpoint_id = :checkpoint_id", + ) + .map_err(Either::Right)?; + + let mut rows = stmt_get_checkpoints + .query(named_params![":limit": limit]) + .map_err(Either::Right)?; + + while let Some(row) = rows.next().map_err(Either::Right)? { + let checkpoint_id = row.get::<_, u32>(0).map_err(Either::Right)?; + let tree_state = row + .get::<_, Option>(1) + .map(|opt| opt.map_or_else(|| TreeState::Empty, |p| TreeState::AtPosition(p.into()))) + .map_err(Either::Right)?; + + let mut mark_removed_rows = stmt_get_checkpoint_marks_removed + .query(named_params![":checkpoint_id": checkpoint_id]) + .map_err(Either::Right)?; + let mut marks_removed = BTreeSet::new(); + while let Some(mr_row) = mark_removed_rows.next().map_err(Either::Right)? { + let mark_removed_position = mr_row + .get::<_, u64>(0) + .map(Position::from) + .map_err(Either::Right)?; + marks_removed.insert(mark_removed_position); + } + + callback( + &BlockHeight::from(checkpoint_id), + &Checkpoint::from_parts(tree_state, marks_removed), + )? + } + + Ok(()) +} + pub(crate) fn update_checkpoint_with( conn: &rusqlite::Transaction<'_>, checkpoint_id: BlockHeight, @@ -426,11 +650,17 @@ pub(crate) fn remove_checkpoint( conn: &rusqlite::Transaction<'_>, checkpoint_id: BlockHeight, ) -> Result<(), Error> { - conn.execute( - "DELETE FROM sapling_tree_checkpoints WHERE checkpoint_id = ?", - [u32::from(checkpoint_id)], - ) - .map_err(Either::Right)?; + // sapling_tree_checkpoints is constructed with `ON DELETE CASCADE` + let mut stmt_delete_checkpoint = conn + .prepare_cached( + "DELETE FROM sapling_tree_checkpoints + WHERE checkpoint_id = :checkpoint_id", + ) + .map_err(Either::Right)?; + + stmt_delete_checkpoint + .execute(named_params![":checkpoint_id": u32::from(checkpoint_id),]) + .map_err(Either::Right)?; Ok(()) } @@ -452,3 +682,62 @@ pub(crate) fn truncate_checkpoints( .map_err(Either::Right)?; Ok(()) } + +#[cfg(test)] +mod tests { + use tempfile::NamedTempFile; + + use incrementalmerkletree::testing::{ + check_append, check_checkpoint_rewind, check_remove_mark, check_rewind_remove_mark, + check_root_hashes, check_witness_consistency, check_witnesses, + }; + use shardtree::ShardTree; + + use super::SqliteShardStore; + use crate::{tests, wallet::init::init_wallet_db, WalletDb}; + + fn new_tree(m: usize) -> ShardTree, 4, 3> { + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + data_file.keep().unwrap(); + + init_wallet_db(&mut db_data, None).unwrap(); + let store = SqliteShardStore::<_, String, 3>::from_connection(db_data.conn).unwrap(); + ShardTree::new(store, m) + } + + #[test] + fn append() { + check_append(new_tree); + } + + #[test] + fn root_hashes() { + check_root_hashes(new_tree); + } + + #[test] + fn witnesses() { + check_witnesses(new_tree); + } + + #[test] + fn witness_consistency() { + check_witness_consistency(new_tree); + } + + #[test] + fn checkpoint_rewind() { + check_checkpoint_rewind(new_tree); + } + + #[test] + fn remove_mark() { + check_remove_mark(new_tree); + } + + #[test] + fn rewind_remove_mark() { + check_rewind_remove_mark(new_tree); + } +} diff --git a/zcash_primitives/src/consensus.rs b/zcash_primitives/src/consensus.rs index dc972f7008..563c698067 100644 --- a/zcash_primitives/src/consensus.rs +++ b/zcash_primitives/src/consensus.rs @@ -627,6 +627,12 @@ pub mod testing { ) }) } + + impl incrementalmerkletree::testing::TestCheckpoint for BlockHeight { + fn from_u64(value: u64) -> Self { + BlockHeight(u32::try_from(value).expect("Test checkpoint ids do not exceed 32 bits")) + } + } } #[cfg(test)] diff --git a/zcash_primitives/src/merkle_tree.rs b/zcash_primitives/src/merkle_tree.rs index 6cda449bcc..0a24b8a1fe 100644 --- a/zcash_primitives/src/merkle_tree.rs +++ b/zcash_primitives/src/merkle_tree.rs @@ -292,6 +292,7 @@ pub mod testing { use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use incrementalmerkletree::frontier::testing::TestNode; use std::io::{self, Read, Write}; + use zcash_encoding::Vector; use super::HashSer; @@ -304,6 +305,23 @@ pub mod testing { writer.write_u64::(self.0) } } + + impl HashSer for String { + fn read(reader: R) -> io::Result { + Vector::read(reader, |r| r.read_u8()).and_then(|xs| { + String::from_utf8(xs).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Not a valid utf8 string: {:?}", e), + ) + }) + }) + } + + fn write(&self, writer: W) -> io::Result<()> { + Vector::write(writer, self.as_bytes(), |w, b| w.write_u8(*b)) + } + } } #[cfg(test)] From 106669d9773b452ec410b76a7ae05e297209315c Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 16 Jun 2023 14:30:53 -0600 Subject: [PATCH 0030/1122] zcash_client_sqlite: Generalize SQLite-backed ShardStore impl to make it reusable for Orchard. --- zcash_client_sqlite/src/lib.rs | 4 +- .../init/migrations/shardtree_support.rs | 1 + .../src/wallet/sapling/commitment_tree.rs | 286 +++++++++++------- 3 files changed, 183 insertions(+), 108 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 9df48d5cac..edf8d78a97 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -633,7 +633,7 @@ impl WalletCommitmentTrees for WalletDb WalletCommitmentTrees for WalletDb>>, { let mut shardtree = ShardTree::new( - SqliteShardStore::from_connection(self.conn.0) + SqliteShardStore::from_connection(self.conn.0, "sapling") .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?, 100, ); diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 6a597d56c7..e5b60ad110 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -94,6 +94,7 @@ impl RusqliteMigration for Migration { let shard_store = SqliteShardStore::<_, sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( transaction, + "sapling", )?; let mut shard_tree: ShardTree< _, diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs index f685350c14..8ce583f9d8 100644 --- a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -15,15 +15,20 @@ use crate::serialization::{read_shard, write_shard_v1}; pub struct SqliteShardStore { pub(crate) conn: C, + table_prefix: &'static str, _hash_type: PhantomData, } impl SqliteShardStore { const SHARD_ROOT_LEVEL: Level = Level::new(SHARD_HEIGHT); - pub(crate) fn from_connection(conn: C) -> Result { + pub(crate) fn from_connection( + conn: C, + table_prefix: &'static str, + ) -> Result { Ok(SqliteShardStore { conn, + table_prefix, _hash_type: PhantomData, }) } @@ -40,39 +45,39 @@ impl<'conn, 'a: 'conn, H: HashSer, const SHARD_HEIGHT: u8> ShardStore &self, shard_root: Address, ) -> Result>, Self::Error> { - get_shard(self.conn, shard_root) + get_shard(self.conn, self.table_prefix, shard_root) } fn last_shard(&self) -> Result>, Self::Error> { - last_shard(self.conn, Self::SHARD_ROOT_LEVEL) + last_shard(self.conn, self.table_prefix, Self::SHARD_ROOT_LEVEL) } fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { - put_shard(self.conn, subtree) + put_shard(self.conn, self.table_prefix, subtree) } fn get_shard_roots(&self) -> Result, Self::Error> { - get_shard_roots(self.conn, Self::SHARD_ROOT_LEVEL) + get_shard_roots(self.conn, self.table_prefix, Self::SHARD_ROOT_LEVEL) } fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { - truncate(self.conn, from) + truncate(self.conn, self.table_prefix, from) } fn get_cap(&self) -> Result, Self::Error> { - get_cap(self.conn) + get_cap(self.conn, self.table_prefix) } fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { - put_cap(self.conn, cap) + put_cap(self.conn, self.table_prefix, cap) } fn min_checkpoint_id(&self) -> Result, Self::Error> { - min_checkpoint_id(self.conn) + min_checkpoint_id(self.conn, self.table_prefix) } fn max_checkpoint_id(&self) -> Result, Self::Error> { - max_checkpoint_id(self.conn) + max_checkpoint_id(self.conn, self.table_prefix) } fn add_checkpoint( @@ -80,32 +85,32 @@ impl<'conn, 'a: 'conn, H: HashSer, const SHARD_HEIGHT: u8> ShardStore checkpoint_id: Self::CheckpointId, checkpoint: Checkpoint, ) -> Result<(), Self::Error> { - add_checkpoint(self.conn, checkpoint_id, checkpoint) + add_checkpoint(self.conn, self.table_prefix, checkpoint_id, checkpoint) } fn checkpoint_count(&self) -> Result { - checkpoint_count(self.conn) + checkpoint_count(self.conn, self.table_prefix) } fn get_checkpoint_at_depth( &self, checkpoint_depth: usize, ) -> Result, Self::Error> { - get_checkpoint_at_depth(self.conn, checkpoint_depth) + get_checkpoint_at_depth(self.conn, self.table_prefix, checkpoint_depth) } fn get_checkpoint( &self, checkpoint_id: &Self::CheckpointId, ) -> Result, Self::Error> { - get_checkpoint(self.conn, *checkpoint_id) + get_checkpoint(self.conn, self.table_prefix, *checkpoint_id) } fn with_checkpoints(&mut self, limit: usize, callback: F) -> Result<(), Self::Error> where F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, { - with_checkpoints(self.conn, limit, callback) + with_checkpoints(self.conn, self.table_prefix, limit, callback) } fn update_checkpoint_with( @@ -116,18 +121,18 @@ impl<'conn, 'a: 'conn, H: HashSer, const SHARD_HEIGHT: u8> ShardStore where F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, { - update_checkpoint_with(self.conn, *checkpoint_id, update) + update_checkpoint_with(self.conn, self.table_prefix, *checkpoint_id, update) } fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { - remove_checkpoint(self.conn, *checkpoint_id) + remove_checkpoint(self.conn, self.table_prefix, *checkpoint_id) } fn truncate_checkpoints( &mut self, checkpoint_id: &Self::CheckpointId, ) -> Result<(), Self::Error> { - truncate_checkpoints(self.conn, *checkpoint_id) + truncate_checkpoints(self.conn, self.table_prefix, *checkpoint_id) } } @@ -142,42 +147,42 @@ impl ShardStore &self, shard_root: Address, ) -> Result>, Self::Error> { - get_shard(&self.conn, shard_root) + get_shard(&self.conn, self.table_prefix, shard_root) } fn last_shard(&self) -> Result>, Self::Error> { - last_shard(&self.conn, Self::SHARD_ROOT_LEVEL) + last_shard(&self.conn, self.table_prefix, Self::SHARD_ROOT_LEVEL) } fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { let tx = self.conn.transaction().map_err(Either::Right)?; - put_shard(&tx, subtree)?; + put_shard(&tx, self.table_prefix, subtree)?; tx.commit().map_err(Either::Right)?; Ok(()) } fn get_shard_roots(&self) -> Result, Self::Error> { - get_shard_roots(&self.conn, Self::SHARD_ROOT_LEVEL) + get_shard_roots(&self.conn, self.table_prefix, Self::SHARD_ROOT_LEVEL) } fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { - truncate(&self.conn, from) + truncate(&self.conn, self.table_prefix, from) } fn get_cap(&self) -> Result, Self::Error> { - get_cap(&self.conn) + get_cap(&self.conn, self.table_prefix) } fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { - put_cap(&self.conn, cap) + put_cap(&self.conn, self.table_prefix, cap) } fn min_checkpoint_id(&self) -> Result, Self::Error> { - min_checkpoint_id(&self.conn) + min_checkpoint_id(&self.conn, self.table_prefix) } fn max_checkpoint_id(&self) -> Result, Self::Error> { - max_checkpoint_id(&self.conn) + max_checkpoint_id(&self.conn, self.table_prefix) } fn add_checkpoint( @@ -186,26 +191,26 @@ impl ShardStore checkpoint: Checkpoint, ) -> Result<(), Self::Error> { let tx = self.conn.transaction().map_err(Either::Right)?; - add_checkpoint(&tx, checkpoint_id, checkpoint)?; + add_checkpoint(&tx, self.table_prefix, checkpoint_id, checkpoint)?; tx.commit().map_err(Either::Right) } fn checkpoint_count(&self) -> Result { - checkpoint_count(&self.conn) + checkpoint_count(&self.conn, self.table_prefix) } fn get_checkpoint_at_depth( &self, checkpoint_depth: usize, ) -> Result, Self::Error> { - get_checkpoint_at_depth(&self.conn, checkpoint_depth) + get_checkpoint_at_depth(&self.conn, self.table_prefix, checkpoint_depth) } fn get_checkpoint( &self, checkpoint_id: &Self::CheckpointId, ) -> Result, Self::Error> { - get_checkpoint(&self.conn, *checkpoint_id) + get_checkpoint(&self.conn, self.table_prefix, *checkpoint_id) } fn with_checkpoints(&mut self, limit: usize, callback: F) -> Result<(), Self::Error> @@ -213,7 +218,7 @@ impl ShardStore F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, { let tx = self.conn.transaction().map_err(Either::Right)?; - with_checkpoints(&tx, limit, callback)?; + with_checkpoints(&tx, self.table_prefix, limit, callback)?; tx.commit().map_err(Either::Right) } @@ -226,14 +231,14 @@ impl ShardStore F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, { let tx = self.conn.transaction().map_err(Either::Right)?; - let result = update_checkpoint_with(&tx, *checkpoint_id, update)?; + let result = update_checkpoint_with(&tx, self.table_prefix, *checkpoint_id, update)?; tx.commit().map_err(Either::Right)?; Ok(result) } fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { let tx = self.conn.transaction().map_err(Either::Right)?; - remove_checkpoint(&tx, *checkpoint_id)?; + remove_checkpoint(&tx, self.table_prefix, *checkpoint_id)?; tx.commit().map_err(Either::Right) } @@ -242,7 +247,7 @@ impl ShardStore checkpoint_id: &Self::CheckpointId, ) -> Result<(), Self::Error> { let tx = self.conn.transaction().map_err(Either::Right)?; - truncate_checkpoints(&tx, *checkpoint_id)?; + truncate_checkpoints(&tx, self.table_prefix, *checkpoint_id)?; tx.commit().map_err(Either::Right) } } @@ -251,12 +256,16 @@ type Error = Either; pub(crate) fn get_shard( conn: &rusqlite::Connection, + table_prefix: &'static str, shard_root: Address, ) -> Result>, Error> { conn.query_row( - "SELECT shard_data - FROM sapling_tree_shards - WHERE shard_index = :shard_index", + &format!( + "SELECT shard_data + FROM {}_tree_shards + WHERE shard_index = :shard_index", + table_prefix + ), named_params![":shard_index": shard_root.index()], |row| row.get::<_, Vec>(0), ) @@ -271,13 +280,17 @@ pub(crate) fn get_shard( pub(crate) fn last_shard( conn: &rusqlite::Connection, + table_prefix: &'static str, shard_root_level: Level, ) -> Result>, Error> { conn.query_row( - "SELECT shard_index, shard_data - FROM sapling_tree_shards - ORDER BY shard_index DESC - LIMIT 1", + &format!( + "SELECT shard_index, shard_data + FROM {}_tree_shards + ORDER BY shard_index DESC + LIMIT 1", + table_prefix + ), [], |row| { let shard_index: u64 = row.get(0)?; @@ -297,6 +310,7 @@ pub(crate) fn last_shard( pub(crate) fn put_shard( conn: &rusqlite::Transaction<'_>, + table_prefix: &'static str, subtree: LocatedPrunableTree, ) -> Result<(), Error> { let subtree_root_hash = subtree @@ -316,13 +330,14 @@ pub(crate) fn put_shard( write_shard_v1(&mut subtree_data, subtree.root()).map_err(Either::Left)?; let mut stmt_put_shard = conn - .prepare_cached( - "INSERT INTO sapling_tree_shards (shard_index, root_hash, shard_data) + .prepare_cached(&format!( + "INSERT INTO {}_tree_shards (shard_index, root_hash, shard_data) VALUES (:shard_index, :root_hash, :shard_data) ON CONFLICT (shard_index) DO UPDATE SET root_hash = :root_hash, shard_data = :shard_data", - ) + table_prefix + )) .map_err(Either::Right)?; stmt_put_shard @@ -338,10 +353,14 @@ pub(crate) fn put_shard( pub(crate) fn get_shard_roots( conn: &rusqlite::Connection, + table_prefix: &'static str, shard_root_level: Level, ) -> Result, Error> { let mut stmt = conn - .prepare("SELECT shard_index FROM sapling_tree_shards ORDER BY shard_index") + .prepare(&format!( + "SELECT shard_index FROM {}_tree_shards ORDER BY shard_index", + table_prefix + )) .map_err(Either::Right)?; let mut rows = stmt.query([]).map_err(Either::Right)?; @@ -355,19 +374,31 @@ pub(crate) fn get_shard_roots( Ok(res) } -pub(crate) fn truncate(conn: &rusqlite::Connection, from: Address) -> Result<(), Error> { +pub(crate) fn truncate( + conn: &rusqlite::Connection, + table_prefix: &'static str, + from: Address, +) -> Result<(), Error> { conn.execute( - "DELETE FROM sapling_tree_shards WHERE shard_index >= ?", + &format!( + "DELETE FROM {}_tree_shards WHERE shard_index >= ?", + table_prefix + ), [from.index()], ) .map_err(Either::Right) .map(|_| ()) } -pub(crate) fn get_cap(conn: &rusqlite::Connection) -> Result, Error> { - conn.query_row("SELECT cap_data FROM sapling_tree_cap", [], |row| { - row.get::<_, Vec>(0) - }) +pub(crate) fn get_cap( + conn: &rusqlite::Connection, + table_prefix: &'static str, +) -> Result, Error> { + conn.query_row( + &format!("SELECT cap_data FROM {}_tree_cap", table_prefix), + [], + |row| row.get::<_, Vec>(0), + ) .optional() .map_err(Either::Right)? .map_or_else( @@ -378,15 +409,17 @@ pub(crate) fn get_cap(conn: &rusqlite::Connection) -> Result( conn: &rusqlite::Connection, + table_prefix: &'static str, cap: PrunableTree, ) -> Result<(), Error> { let mut stmt = conn - .prepare_cached( - "INSERT INTO sapling_tree_cap (cap_id, cap_data) - VALUES (0, :cap_data) - ON CONFLICT (cap_id) DO UPDATE - SET cap_data = :cap_data", - ) + .prepare_cached(&format!( + "INSERT INTO {}_tree_cap (cap_id, cap_data) + VALUES (0, :cap_data) + ON CONFLICT (cap_id) DO UPDATE + SET cap_data = :cap_data", + table_prefix + )) .map_err(Either::Right)?; let mut cap_data = vec![]; @@ -396,9 +429,15 @@ pub(crate) fn put_cap( Ok(()) } -pub(crate) fn min_checkpoint_id(conn: &rusqlite::Connection) -> Result, Error> { +pub(crate) fn min_checkpoint_id( + conn: &rusqlite::Connection, + table_prefix: &'static str, +) -> Result, Error> { conn.query_row( - "SELECT MIN(checkpoint_id) FROM sapling_tree_checkpoints", + &format!( + "SELECT MIN(checkpoint_id) FROM {}_tree_checkpoints", + table_prefix + ), [], |row| { row.get::<_, Option>(0) @@ -408,9 +447,15 @@ pub(crate) fn min_checkpoint_id(conn: &rusqlite::Connection) -> Result Result, Error> { +pub(crate) fn max_checkpoint_id( + conn: &rusqlite::Connection, + table_prefix: &'static str, +) -> Result, Error> { conn.query_row( - "SELECT MAX(checkpoint_id) FROM sapling_tree_checkpoints", + &format!( + "SELECT MAX(checkpoint_id) FROM {}_tree_checkpoints", + table_prefix + ), [], |row| { row.get::<_, Option>(0) @@ -422,14 +467,16 @@ pub(crate) fn max_checkpoint_id(conn: &rusqlite::Connection) -> Result, + table_prefix: &'static str, checkpoint_id: BlockHeight, checkpoint: Checkpoint, ) -> Result<(), Error> { let mut stmt_insert_checkpoint = conn - .prepare_cached( - "INSERT INTO sapling_tree_checkpoints (checkpoint_id, position) + .prepare_cached(&format!( + "INSERT INTO {}_tree_checkpoints (checkpoint_id, position) VALUES (:checkpoint_id, :position)", - ) + table_prefix + )) .map_err(Either::Right)?; stmt_insert_checkpoint @@ -439,10 +486,13 @@ pub(crate) fn add_checkpoint( ]) .map_err(Either::Right)?; - let mut stmt_insert_mark_removed = conn.prepare_cached( - "INSERT INTO sapling_tree_checkpoint_marks_removed (checkpoint_id, mark_removed_position) - VALUES (:checkpoint_id, :position)", - ).map_err(Either::Right)?; + let mut stmt_insert_mark_removed = conn + .prepare_cached(&format!( + "INSERT INTO {}_tree_checkpoint_marks_removed (checkpoint_id, mark_removed_position) + VALUES (:checkpoint_id, :position)", + table_prefix + )) + .map_err(Either::Right)?; for pos in checkpoint.marks_removed() { stmt_insert_mark_removed @@ -456,22 +506,31 @@ pub(crate) fn add_checkpoint( Ok(()) } -pub(crate) fn checkpoint_count(conn: &rusqlite::Connection) -> Result { - conn.query_row("SELECT COUNT(*) FROM sapling_tree_checkpoints", [], |row| { - row.get::<_, usize>(0) - }) +pub(crate) fn checkpoint_count( + conn: &rusqlite::Connection, + table_prefix: &'static str, +) -> Result { + conn.query_row( + &format!("SELECT COUNT(*) FROM {}_tree_checkpoints", table_prefix), + [], + |row| row.get::<_, usize>(0), + ) .map_err(Either::Right) } pub(crate) fn get_checkpoint( conn: &rusqlite::Connection, + table_prefix: &'static str, checkpoint_id: BlockHeight, ) -> Result, Error> { let checkpoint_position = conn .query_row( - "SELECT position - FROM sapling_tree_checkpoints + &format!( + "SELECT position + FROM {}_tree_checkpoints WHERE checkpoint_id = ?", + table_prefix + ), [u32::from(checkpoint_id)], |row| { row.get::<_, Option>(0) @@ -485,11 +544,12 @@ pub(crate) fn get_checkpoint( .map(|pos_opt| { let mut marks_removed = BTreeSet::new(); let mut stmt = conn - .prepare_cached( + .prepare_cached(&format!( "SELECT mark_removed_position - FROM sapling_tree_checkpoint_marks_removed + FROM {}_tree_checkpoint_marks_removed WHERE checkpoint_id = ?", - ) + table_prefix + )) .map_err(Either::Right)?; let mut mark_removed_rows = stmt .query([u32::from(checkpoint_id)]) @@ -513,6 +573,7 @@ pub(crate) fn get_checkpoint( pub(crate) fn get_checkpoint_at_depth( conn: &rusqlite::Connection, + table_prefix: &'static str, checkpoint_depth: usize, ) -> Result, Error> { if checkpoint_depth == 0 { @@ -521,11 +582,14 @@ pub(crate) fn get_checkpoint_at_depth( let checkpoint_parts = conn .query_row( - "SELECT checkpoint_id, position - FROM sapling_tree_checkpoints - ORDER BY checkpoint_id DESC - LIMIT 1 - OFFSET :offset", + &format!( + "SELECT checkpoint_id, position + FROM {}_tree_checkpoints + ORDER BY checkpoint_id DESC + LIMIT 1 + OFFSET :offset", + table_prefix + ), named_params![":offset": checkpoint_depth - 1], |row| { let checkpoint_id: u32 = row.get(0)?; @@ -543,11 +607,12 @@ pub(crate) fn get_checkpoint_at_depth( .map(|(checkpoint_id, pos_opt)| { let mut marks_removed = BTreeSet::new(); let mut stmt = conn - .prepare_cached( + .prepare_cached(&format!( "SELECT mark_removed_position - FROM sapling_tree_checkpoint_marks_removed + FROM {}_tree_checkpoint_marks_removed WHERE checkpoint_id = ?", - ) + table_prefix + )) .map_err(Either::Right)?; let mut mark_removed_rows = stmt .query([u32::from(checkpoint_id)]) @@ -574,6 +639,7 @@ pub(crate) fn get_checkpoint_at_depth( pub(crate) fn with_checkpoints( conn: &rusqlite::Transaction<'_>, + table_prefix: &'static str, limit: usize, mut callback: F, ) -> Result<(), Error> @@ -581,19 +647,21 @@ where F: FnMut(&BlockHeight, &Checkpoint) -> Result<(), Error>, { let mut stmt_get_checkpoints = conn - .prepare_cached( + .prepare_cached(&format!( "SELECT checkpoint_id, position - FROM sapling_tree_checkpoints + FROM {}_tree_checkpoints LIMIT :limit", - ) + table_prefix + )) .map_err(Either::Right)?; let mut stmt_get_checkpoint_marks_removed = conn - .prepare_cached( + .prepare_cached(&format!( "SELECT mark_removed_position - FROM sapling_tree_checkpoint_marks_removed + FROM {}_tree_checkpoint_marks_removed WHERE checkpoint_id = :checkpoint_id", - ) + table_prefix + )) .map_err(Either::Right)?; let mut rows = stmt_get_checkpoints @@ -630,16 +698,17 @@ where pub(crate) fn update_checkpoint_with( conn: &rusqlite::Transaction<'_>, + table_prefix: &'static str, checkpoint_id: BlockHeight, update: F, ) -> Result where F: Fn(&mut Checkpoint) -> Result<(), Error>, { - if let Some(mut c) = get_checkpoint(conn, checkpoint_id)? { + if let Some(mut c) = get_checkpoint(conn, table_prefix, checkpoint_id)? { update(&mut c)?; - remove_checkpoint(conn, checkpoint_id)?; - add_checkpoint(conn, checkpoint_id, c)?; + remove_checkpoint(conn, table_prefix, checkpoint_id)?; + add_checkpoint(conn, table_prefix, checkpoint_id, c)?; Ok(true) } else { Ok(false) @@ -648,14 +717,17 @@ where pub(crate) fn remove_checkpoint( conn: &rusqlite::Transaction<'_>, + table_prefix: &'static str, checkpoint_id: BlockHeight, ) -> Result<(), Error> { - // sapling_tree_checkpoints is constructed with `ON DELETE CASCADE` + // cascading delete here obviates the need to manually delete from + // `tree_checkpoint_marks_removed` let mut stmt_delete_checkpoint = conn - .prepare_cached( - "DELETE FROM sapling_tree_checkpoints + .prepare_cached(&format!( + "DELETE FROM {}_tree_checkpoints WHERE checkpoint_id = :checkpoint_id", - ) + table_prefix + )) .map_err(Either::Right)?; stmt_delete_checkpoint @@ -667,19 +739,20 @@ pub(crate) fn remove_checkpoint( pub(crate) fn truncate_checkpoints( conn: &rusqlite::Transaction<'_>, + table_prefix: &'static str, checkpoint_id: BlockHeight, ) -> Result<(), Error> { + // cascading delete here obviates the need to manually delete from + // `tree_checkpoint_marks_removed` conn.execute( - "DELETE FROM sapling_tree_checkpoints WHERE checkpoint_id >= ?", + &format!( + "DELETE FROM {}_tree_checkpoints WHERE checkpoint_id >= ?", + table_prefix + ), [u32::from(checkpoint_id)], ) .map_err(Either::Right)?; - conn.execute( - "DELETE FROM sapling_tree_checkpoint_marks_removed WHERE checkpoint_id >= ?", - [u32::from(checkpoint_id)], - ) - .map_err(Either::Right)?; Ok(()) } @@ -702,7 +775,8 @@ mod tests { data_file.keep().unwrap(); init_wallet_db(&mut db_data, None).unwrap(); - let store = SqliteShardStore::<_, String, 3>::from_connection(db_data.conn).unwrap(); + let store = + SqliteShardStore::<_, String, 3>::from_connection(db_data.conn, "sapling").unwrap(); ShardTree::new(store, m) } From 547634e210b7817764e1f107206234d79d9ef0e1 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 16 Jun 2023 14:42:36 -0600 Subject: [PATCH 0031/1122] zcash_client_sqlite: Move the SqliteShardStore implementation out of the `wallet::sapling` module. --- zcash_client_sqlite/src/lib.rs | 2 +- zcash_client_sqlite/src/wallet.rs | 1 + zcash_client_sqlite/src/wallet/{sapling => }/commitment_tree.rs | 0 .../src/wallet/init/migrations/shardtree_support.rs | 2 +- zcash_client_sqlite/src/wallet/sapling.rs | 2 -- 5 files changed, 3 insertions(+), 4 deletions(-) rename zcash_client_sqlite/src/wallet/{sapling => }/commitment_tree.rs (100%) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index edf8d78a97..1c4e887d2e 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -66,7 +66,7 @@ use zcash_client_backend::{ DecryptedOutput, TransferType, }; -use crate::{error::SqliteClientError, wallet::sapling::commitment_tree::SqliteShardStore}; +use crate::{error::SqliteClientError, wallet::commitment_tree::SqliteShardStore}; #[cfg(feature = "unstable")] use { diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 7ecbc080c7..757fde8bf9 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -104,6 +104,7 @@ use { }, }; +pub(crate) mod commitment_tree; pub mod init; pub(crate) mod sapling; diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs similarity index 100% rename from zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs rename to zcash_client_sqlite/src/wallet/commitment_tree.rs diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index e5b60ad110..0d292adb12 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -19,8 +19,8 @@ use zcash_primitives::{ }; use crate::wallet::{ + commitment_tree::SqliteShardStore, init::{migrations::received_notes_nullable_nf, WalletMigrationError}, - sapling::commitment_tree::SqliteShardStore, }; pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index c763534d04..7b32691402 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -22,8 +22,6 @@ use crate::{error::SqliteClientError, NoteId}; use super::memo_repr; -pub(crate) mod commitment_tree; - /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedSaplingOutput { fn index(&self) -> usize; From ba709177d3725ec81cc5ce41f349ade9028fd60a Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 29 Jun 2023 14:28:12 -0600 Subject: [PATCH 0032/1122] Reorganize Sapling and Orchard note commitment tree sizes in CompactBlock. We move thes fields out into a separate BlockMetadata struct to ensure that future additions to block metadata are structurally separated from future additions to block data. --- .../proto/compact_formats.proto | 25 +++++++++------ .../src/proto/compact_formats.rs | 22 +++++++++---- zcash_client_backend/src/proto/service.rs | 13 ++++++++ zcash_client_backend/src/welding_rig.rs | 31 ++++++++++++------- zcash_client_sqlite/src/lib.rs | 16 +++++++--- 5 files changed, 75 insertions(+), 32 deletions(-) diff --git a/zcash_client_backend/proto/compact_formats.proto b/zcash_client_backend/proto/compact_formats.proto index eac2b2f2f3..740d7f7f3f 100644 --- a/zcash_client_backend/proto/compact_formats.proto +++ b/zcash_client_backend/proto/compact_formats.proto @@ -10,20 +10,27 @@ option swift_prefix = ""; // Remember that proto3 fields are all optional. A field that is not present will be set to its zero value. // bytes fields of hashes are in canonical little-endian format. +// BlockMetadata represents information about a block that may not be +// represented directly in the block data, but is instead derived from chain +// data or other external sources. +message BlockMetadata { + uint32 saplingCommitmentTreeSize = 1; // the size of the Sapling note commitment tree as of the end of this block + uint32 orchardCommitmentTreeSize = 2; // the size of the Orchard note commitment tree as of the end of this block +} + // CompactBlock is a packaging of ONLY the data from a block that's needed to: // 1. Detect a payment to your shielded Sapling address // 2. Detect a spend of your shielded Sapling notes // 3. Update your witnesses to generate new Sapling spend proofs. message CompactBlock { - uint32 protoVersion = 1; // the version of this wire format, for storage - uint64 height = 2; // the height of this block - bytes hash = 3; // the ID (hash) of this block, same as in block explorers - bytes prevHash = 4; // the ID (hash) of this block's predecessor - uint32 time = 5; // Unix epoch time when the block was mined - bytes header = 6; // (hash, prevHash, and time) OR (full header) - repeated CompactTx vtx = 7; // zero or more compact transactions from this block - uint32 saplingCommitmentTreeSize = 8; // the size of the Sapling note commitment tree as of the end of this block - uint32 orchardCommitmentTreeSize = 9; // the size of the Orchard note commitment tree as of the end of this block + uint32 protoVersion = 1; // the version of this wire format, for storage + uint64 height = 2; // the height of this block + bytes hash = 3; // the ID (hash) of this block, same as in block explorers + bytes prevHash = 4; // the ID (hash) of this block's predecessor + uint32 time = 5; // Unix epoch time when the block was mined + bytes header = 6; // (hash, prevHash, and time) OR (full header) + repeated CompactTx vtx = 7; // zero or more compact transactions from this block + BlockMetadata blockMetadata = 8; // information about this block derived from the chain or other sources } // CompactTx contains the minimum information for a wallet to know if this transaction diff --git a/zcash_client_backend/src/proto/compact_formats.rs b/zcash_client_backend/src/proto/compact_formats.rs index c8d45173c4..bf023eacfc 100644 --- a/zcash_client_backend/src/proto/compact_formats.rs +++ b/zcash_client_backend/src/proto/compact_formats.rs @@ -1,3 +1,16 @@ +/// BlockMetadata represents information about a block that may not be +/// represented directly in the block data, but is instead derived from chain +/// data or other external sources. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockMetadata { + /// the size of the Sapling note commitment tree as of the end of this block + #[prost(uint32, tag = "1")] + pub sapling_commitment_tree_size: u32, + /// the size of the Orchard note commitment tree as of the end of this block + #[prost(uint32, tag = "2")] + pub orchard_commitment_tree_size: u32, +} /// CompactBlock is a packaging of ONLY the data from a block that's needed to: /// 1. Detect a payment to your shielded Sapling address /// 2. Detect a spend of your shielded Sapling notes @@ -26,12 +39,9 @@ pub struct CompactBlock { /// zero or more compact transactions from this block #[prost(message, repeated, tag = "7")] pub vtx: ::prost::alloc::vec::Vec, - /// the size of the Sapling note commitment tree as of the end of this block - #[prost(uint32, tag = "8")] - pub sapling_commitment_tree_size: u32, - /// the size of the Orchard note commitment tree as of the end of this block - #[prost(uint32, tag = "9")] - pub orchard_commitment_tree_size: u32, + /// information about this block derived from the chain or other sources + #[prost(message, optional, tag = "8")] + pub block_metadata: ::core::option::Option, } /// CompactTx contains the minimum information for a wallet to know if this transaction /// is relevant to it (either pays to it or spends from it) via shielded elements diff --git a/zcash_client_backend/src/proto/service.rs b/zcash_client_backend/src/proto/service.rs index 38b15abdbf..581762bb3b 100644 --- a/zcash_client_backend/src/proto/service.rs +++ b/zcash_client_backend/src/proto/service.rs @@ -1,3 +1,16 @@ +/// BlockMetadata represents information about a block that may not be +/// represented directly in the block data, but is instead derived from chain +/// data or other external sources. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockMetadata { + /// the size of the Sapling note commitment tree as of the end of this block + #[prost(uint32, tag = "1")] + pub sapling_commitment_tree_size: u32, + /// the size of the Orchard note commitment tree as of the end of this block + #[prost(uint32, tag = "2")] + pub orchard_commitment_tree_size: u32, +} /// A BlockID message contains identifiers to select a block: a height or a /// hash. Specification by hash is not implemented, but may be in the future. #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index 3266d40594..a9e7d3f56b 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -218,16 +218,21 @@ pub(crate) fn scan_block_with_runner< // to use it. `block.sapling_commitment_tree_size` is expected to be correct as of the end of // the block, and we can't have a note of ours in a block with no outputs so treating the zero // default value from the protobuf as `None` is always correct. - let mut sapling_tree_position = if block.sapling_commitment_tree_size == 0 { - initial_commitment_tree_meta.map(|m| (m.sapling_tree_size() + 1).into()) - } else { - let end_position_exclusive = Position::from(u64::from(block.sapling_commitment_tree_size)); + let mut sapling_tree_position = if let Some(sapling_tree_size) = block + .block_metadata + .as_ref() + .map(|m| m.sapling_commitment_tree_size) + .filter(|s| *s != 0) + { + let end_position_exclusive = Position::from(u64::from(sapling_tree_size)); let output_count = block .vtx .iter() .map(|tx| u64::try_from(tx.outputs.len()).unwrap()) .sum(); Some(end_position_exclusive - output_count) + } else { + initial_commitment_tree_meta.map(|m| (m.sapling_tree_size() + 1).into()) }; for tx in block.vtx.into_iter() { @@ -404,11 +409,10 @@ pub(crate) fn scan_block_with_runner< block_hash, block_time: block.time, transactions: wtxs, - sapling_commitment_tree_size: if block.sapling_commitment_tree_size == 0 { - None - } else { - Some(block.sapling_commitment_tree_size) - }, + sapling_commitment_tree_size: block + .block_metadata + .map(|m| m.sapling_commitment_tree_size) + .filter(|s| *s != 0), sapling_commitments: sapling_note_commitments, }) } @@ -439,7 +443,7 @@ mod tests { use crate::{ data_api::chain::CommitmentTreeMeta, proto::compact_formats::{ - CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, + BlockMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }, scan::BatchRunner, }; @@ -547,8 +551,11 @@ mod tests { cb.vtx.push(tx); } - cb.sapling_commitment_tree_size = initial_sapling_tree_size - + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); + cb.block_metadata = Some(BlockMetadata { + sapling_commitment_tree_size: initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), + ..Default::default() + }); cb } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 1c4e887d2e..60447a55ec 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -964,7 +964,7 @@ mod tests { data_api::{WalletRead, WalletWrite}, keys::{sapling, UnifiedFullViewingKey}, proto::compact_formats::{ - CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, + BlockMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }, }; @@ -1109,8 +1109,11 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); - cb.sapling_commitment_tree_size = initial_sapling_tree_size - + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); + cb.block_metadata = Some(BlockMetadata { + sapling_commitment_tree_size: initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), + ..Default::default() + }); (cb, note.nf(&dfvk.fvk().vk.nk, 0)) } @@ -1197,8 +1200,11 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); - cb.sapling_commitment_tree_size = initial_sapling_tree_size - + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); + cb.block_metadata = Some(BlockMetadata { + sapling_commitment_tree_size: initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), + ..Default::default() + }); cb } From d65b129b43cc29b3d5b51de61f860fab23c60544 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 29 Jun 2023 16:26:22 -0600 Subject: [PATCH 0033/1122] Apply changelog, documentation & style suggestions from code review Co-authored-by: str4d --- zcash_client_backend/CHANGELOG.md | 77 +++++++++++++++---- zcash_client_backend/src/data_api.rs | 4 + zcash_client_backend/src/data_api/chain.rs | 49 +++++++----- zcash_client_backend/src/data_api/wallet.rs | 4 + .../src/data_api/wallet/input_selection.rs | 6 +- zcash_client_backend/src/welding_rig.rs | 31 ++++++++ zcash_client_sqlite/CHANGELOG.md | 9 +++ zcash_client_sqlite/src/error.rs | 3 +- zcash_client_sqlite/src/lib.rs | 18 +++-- zcash_client_sqlite/src/serialization.rs | 18 +++-- zcash_client_sqlite/src/wallet.rs | 2 +- .../src/wallet/commitment_tree.rs | 6 +- .../init/migrations/shardtree_support.rs | 18 ++--- zcash_client_sqlite/src/wallet/sapling.rs | 2 +- zcash_primitives/CHANGELOG.md | 3 + 15 files changed, 181 insertions(+), 69 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 28ab398643..0110fa88b1 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -7,30 +7,73 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added -- `impl Eq for address::RecipientAddress` -- `impl Eq for zip321::{Payment, TransactionRequest}` -- `data_api::NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` -- `WalletWrite::put_block` -- `impl Debug` for `{data_api::wallet::input_selection::Proposal, wallet::ReceivedSaplingNote} +- `impl Eq for zcash_client_backend::address::RecipientAddress` +- `impl Eq for zcash_client_backend::zip321::{Payment, TransactionRequest}` +- `impl Debug` for `zcash_client_backend::{data_api::wallet::input_selection::Proposal, wallet::ReceivedSaplingNote}` +- `zcash_client_backend::data_api`: + - `WalletRead::{fully_scanned_height, suggest_scan_ranges}` + - `WalletWrite::put_block` + - `WalletCommitmentTrees` + - `testing::MockWalletDb::new` + - `NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` + - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}`: +- `zcash_client_backend::wallet::WalletSaplingOutput::note_commitment_tree_position` +- `zcash_client_backend::welding_rig::SyncError` ### Changed - MSRV is now 1.65.0. - Bumped dependencies to `hdwallet 0.4`, `zcash_primitives 0.12`, `zcash_note_encryption 0.4`, `incrementalmerkletree 0.4`, `orchard 0.5`, `bs58 0.5` -- `WalletRead::get_memo` now returns `Result, Self::Error>` - instead of `Result` in order to make representable - wallet states where the full note plaintext is not available. -- `WalletRead::get_nullifiers` has been renamed to `WalletRead::get_sapling_nullifiers` - and its signature has changed; it now subsumes the removed `WalletRead::get_all_nullifiers`. -- `wallet::SpendableNote` has been renamed to `wallet::ReceivedSaplingNote`. -- `data_api::chain::scan_cached_blocks` now takes a `from_height` argument that - permits the caller to control the starting position of the scan range. -- `WalletWrite::advance_by_block` has been replaced by `WalletWrite::put_block` - to reflect the semantic change that scanning is no longer a linear operation. +- `zcash_client_backend::data_api`: + - `WalletRead::get_memo` now returns `Result, Self::Error>` + instead of `Result` in order to make representable + wallet states where the full note plaintext is not available. + - `WalletRead::get_nullifiers` has been renamed to `WalletRead::get_sapling_nullifiers` + and its signature has changed; it now subsumes the removed `WalletRead::get_all_nullifiers`. + - `chain::scan_cached_blocks` now takes a `from_height` argument that + permits the caller to control the starting position of the scan range. + - A new `CommitmentTree` variant has been added to `data_api::error::Error` + - `data_api::wallet::{create_spend_to_address, create_proposed_transaction, + shield_transparent_funds}` all now require that `WalletCommitmentTrees` be + implemented for the type passed to them for the `wallet_db` parameter. + - `data_api::wallet::create_proposed_transaction` now takes an additional + `min_confirmations` argument. + - A new `Sync` variant has been added to `data_api::chain::error::Error`. + - A new `SyncRequired` variant has been added to `data_api::wallet::input_selection::InputSelectorError`. +- `zcash_client_backend::wallet`: + - `SpendableNote` has been renamed to `ReceivedSaplingNote`. + - Arguments to `WalletSaplingOutput::from_parts` have changed. +- `zcash_client_backend::data_api::wallet::input_selection::InputSelector`: + - Arguments to `{propose_transaction, propose_shielding}` have changed. +- `zcash_client_backend::wallet::ReceivedSaplingNote::note_commitment_tree_position` + has replaced the `witness` field in the same struct. +- `zcash_client_backend::welding_rig::ScanningKey::sapling_nf` has been changed to + take a note position instead of an incremental witness for the note. +- Arguments to `zcash_client_backend::welding_rig::scan_block` have changed. This + method now takes an optional `CommitmentTreeMeta` argument instead of a base commitment + tree and incremental witnesses for each previously-known note. + ### Removed -- `WalletRead::get_all_nullifiers` -- `WalletWrite::advance_by_block` +- `zcash_client_backend::data_api`: + - `WalletRead::get_all_nullifiers` + - `WalletRead::{get_commitment_tree, get_witnesses}` (use + `WalletRead::fully_scanned_height` instead). + - `WalletWrite::advance_by_block` (use `WalletWrite::put_block` instead). + - The `commitment_tree` and `transactions` properties of the `PrunedBlock` + struct are now owned values instead of references, making this a fully owned type. + It is now parameterized by the nullifier type instead of by a lifetime for the + fields that were previously references. In addition, two new properties, + `sapling_commitment_tree_size` and `sapling_commitments` have been added. + - `testing::MockWalletDb`, which is available under the `test-dependencies` + feature flag, has been modified by the addition of a `sapling_tree` property. + - `wallet::input_selection`: + - `Proposal::target_height` (use `Proposal::min_target_height` instead). +- `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}` + have been removed as individual incremental witnesses are no longer tracked on a + per-note basis. The global note commitment tree for the wallet should be used + to obtain witnesses for spend operations instead. + ## [0.9.0] - 2023-04-28 ### Added diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index eafa0e89c9..d1031e5701 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -438,6 +438,10 @@ pub trait WalletWrite: WalletRead { ) -> Result; } +/// This trait describes a capability for manipulating wallet note commitment trees. +/// +/// At present, this only serves the Sapling protocol, but it will be modified to +/// also provide operations related to Orchard note commitment trees in the future. pub trait WalletCommitmentTrees { type Error; type SaplingShardStore<'a>: ShardStore< diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index dbec6768d1..66355b06aa 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -101,16 +101,20 @@ use crate::{ pub mod error; use error::{ChainError, Error}; +/// Metadata describing the sizes of the zcash note commitment trees as of a particular block. pub struct CommitmentTreeMeta { sapling_tree_size: u64, //TODO: orchard_tree_size: u64 } impl CommitmentTreeMeta { + /// Constructs a new [`CommitmentTreeMeta`] value from its constituent parts. pub fn from_parts(sapling_tree_size: u64) -> Self { Self { sapling_tree_size } } + /// Returns the size of the Sapling note commitment tree as of the block that this + /// [`CommitmentTreeMeta`] describes. pub fn sapling_tree_size(&self) -> u64 { self.sapling_tree_size } @@ -199,23 +203,19 @@ where /// Scans at most `limit` new blocks added to the block source for any transactions received by the /// tracked accounts. /// -/// This function will return without error after scanning at most `limit` new blocks, to enable -/// the caller to update their UI with scanning progress. Repeatedly calling this function will -/// process sequential ranges of blocks, and is equivalent to calling `scan_cached_blocks` and -/// passing `None` for the optional `limit` value. -/// -/// This function pays attention only to cached blocks with heights greater than the highest -/// scanned block in `data`. Cached blocks with lower heights are not verified against -/// previously-scanned blocks. In particular, this function **assumes** that the caller is handling -/// rollbacks. +/// If the `from_height` argument is not `None`, then the block source will begin requesting blocks +/// from the provided block source at the specified height; if `from_height` is `None then this +/// will begin scanning at first block after the position to which the wallet has previously +/// fully scanned the chain, thereby beginning or continuing a linear scan over all blocks. /// -/// For brand-new light client databases, this function starts scanning from the Sapling activation -/// height. This height can be fast-forwarded to a more recent block by initializing the client -/// database with a starting block (for example, calling `init_blocks_table` before this function -/// if using `zcash_client_sqlite`). +/// This function will return without error after scanning at most `limit` new blocks, to enable +/// the caller to update their UI with scanning progress. Repeatedly calling this function with +/// `from_height == None` will process sequential ranges of blocks. /// -/// Scanned blocks are required to be height-sequential. If a block is missing from the block -/// source, an error will be returned with cause [`error::Cause::BlockHeightDiscontinuity`]. +/// For brand-new light client databases, if `from_height == None` this function starts scanning +/// from the Sapling activation height. This height can be fast-forwarded to a more recent block by +/// initializing the client database with a starting block (for example, calling +/// `init_blocks_table` before this function if using `zcash_client_sqlite`). #[tracing::instrument(skip(params, block_source, data_db))] #[allow(clippy::type_complexity)] pub fn scan_cached_blocks( @@ -260,7 +260,7 @@ where ); // Start at either the provided height, or where we synced up to previously. - let (from_height, commitment_tree_meta) = from_height.map_or_else( + let (scan_from, commitment_tree_meta) = from_height.map_or_else( || { data_db.fully_scanned_height().map_or_else( |e| Err(Error::Wallet(e)), @@ -273,7 +273,7 @@ where )?; block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - from_height, + scan_from, limit, |block: CompactBlock| { add_block_to_runner(params, block, &mut batch_runner); @@ -283,10 +283,22 @@ where batch_runner.flush(); + let mut last_scanned = None; block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - from_height, + scan_from, limit, |block: CompactBlock| { + // block heights should be sequential within a single scan range + let block_height = block.height(); + if let Some(h) = last_scanned { + if block_height != h + 1 { + return Err(Error::Chain(ChainError::block_height_discontinuity( + block.height(), + h, + ))); + } + } + let pruned_block = scan_block_with_runner( params, block, @@ -312,6 +324,7 @@ where data_db.put_block(pruned_block).map_err(Error::Wallet)?; + last_scanned = Some(block_height); Ok(()) }, )?; diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index d37a033295..a475289209 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -122,6 +122,10 @@ where /// spent. A value of 10 confirmations is recommended and 0-conf transactions are /// not supported. /// +/// # Panics +/// +/// Panics if `min_confirmations == 0`; 0-conf transactions are not supported. +/// /// # Examples /// /// ``` diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 5017ae3350..90c7013360 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -62,7 +62,9 @@ impl fmt::Display for InputSelectorError write!(f, "No chain data is available."), + InputSelectorError::SyncRequired => { + write!(f, "Insufficient chain data is available, sync required.") + } } } } @@ -135,7 +137,7 @@ impl Debug for Proposal { .field("min_target_height", &self.min_target_height) .field("min_anchor_height", &self.min_anchor_height) .field("is_shielding", &self.is_shielding) - .finish() + .finish_non_exhaustive() } } diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index a9e7d3f56b..a8a98069ea 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -672,6 +672,22 @@ mod tests { assert_eq!(tx.sapling_outputs[0].index(), 0); assert_eq!(tx.sapling_outputs[0].account(), AccountId::from(0)); assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); + + assert_eq!( + pruned_block + .sapling_commitments + .iter() + .map(|(_, retention)| *retention) + .collect::>(), + vec![ + Retention::Ephemeral, + Retention::Marked, + Retention::Checkpoint { + id: pruned_block.block_height, + is_marked: false + } + ] + ); } go(false); @@ -714,5 +730,20 @@ mod tests { assert_eq!(tx.sapling_spends[0].index(), 0); assert_eq!(tx.sapling_spends[0].nf(), &nf); assert_eq!(tx.sapling_spends[0].account(), account); + + assert_eq!( + pruned_block + .sapling_commitments + .iter() + .map(|(_, retention)| *retention) + .collect::>(), + vec![ + Retention::Ephemeral, + Retention::Checkpoint { + id: pruned_block.block_height, + is_marked: false + } + ] + ); } } diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index d72fe90ac0..b1e332913d 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -6,10 +6,19 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- `zcash_client_sqlite::serialization` Serialization formats for data stored + as SQLite BLOBs in the wallet database. + ### Changed - MSRV is now 1.65.0. - Bumped dependencies to `hdwallet 0.4`, `incrementalmerkletree 0.4`, `bs58 0.5`, `zcash_primitives 0.12` +- A `CommitmentTree` variant has been added to `zcash_client_sqlite::wallet::init::WalletMigrationError` +- `min_confirmations` parameter values are now more strongly enforced. Previously, + a note could be spent with fewer than `min_confirmations` confirmations if the + wallet did not contain enough observed blocks to satisfy the `min_confirmations` + value specified; this situation is now treated as an error. ### Removed - The empty `wallet::transact` module has been removed. diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 87f88b8441..6eb0939f24 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -78,7 +78,8 @@ pub enum SqliteClientError { #[cfg(feature = "transparent-inputs")] AddressNotRecognized(TransparentAddress), - /// An error occurred in inserting data into one of the wallet's note commitment trees. + /// An error occurred in inserting data into or accessing data from one of the wallet's note + /// commitment trees. CommitmentTree(ShardTreeError>), } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 60447a55ec..9c1ac261a7 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -85,6 +85,8 @@ pub mod wallet; /// this delta from the chain tip to be pruned. pub(crate) const PRUNING_HEIGHT: u32 = 100; +pub(crate) const SAPLING_TABLES_PREFIX: &'static str = "sapling"; + /// A newtype wrapper for sqlite primary key values for the notes /// table. #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] @@ -396,10 +398,9 @@ impl WalletWrite for WalletDb ) -> Result, Self::Error> { self.transactionally(|wdb| { // Insert the block into the database. - let block_height = block.block_height; wallet::put_block( wdb.conn.0, - block_height, + block.block_height, block.block_hash, block.block_time, block.sapling_commitment_tree_size.map(|s| s.into()), @@ -423,18 +424,19 @@ impl WalletWrite for WalletDb } } + let sapling_commitments_len = block.sapling_commitments.len(); let mut sapling_commitments = block.sapling_commitments.into_iter(); wdb.with_sapling_tree_mut::<_, _, SqliteClientError>(move |sapling_tree| { if let Some(sapling_tree_size) = block.sapling_commitment_tree_size { let start_position = Position::from(u64::from(sapling_tree_size)) - - u64::try_from(sapling_commitments.len()).unwrap(); + - u64::try_from(sapling_commitments_len).unwrap(); sapling_tree.batch_insert(start_position, &mut sapling_commitments)?; } Ok(()) })?; // Update now-expired transactions that didn't get mined. - wallet::update_expired_notes(wdb.conn.0, block_height)?; + wallet::update_expired_notes(wdb.conn.0, block.block_height)?; Ok(wallet_note_ids) }) @@ -633,10 +635,10 @@ impl WalletCommitmentTrees for WalletDb WalletCommitmentTrees for WalletDb>>, { let mut shardtree = ShardTree::new( - SqliteShardStore::from_connection(self.conn.0, "sapling") + SqliteShardStore::from_connection(self.conn.0, SAPLING_TABLES_PREFIX) .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?, - 100, + PRUNING_HEIGHT.try_into().unwrap(), ); let result = callback(&mut shardtree)?; diff --git a/zcash_client_sqlite/src/serialization.rs b/zcash_client_sqlite/src/serialization.rs index 99cb90dd82..eb11764659 100644 --- a/zcash_client_sqlite/src/serialization.rs +++ b/zcash_client_sqlite/src/serialization.rs @@ -14,10 +14,11 @@ const NIL_TAG: u8 = 0; const LEAF_TAG: u8 = 1; const PARENT_TAG: u8 = 2; -pub fn write_shard_v1( - writer: &mut W, - tree: &PrunableTree, -) -> io::Result<()> { +/// Writes a [`PrunableTree`] to the provided [`Write`] instance. +/// +/// This is the primary method used for ShardTree shard persistence. It writes a version identifier +/// for the most-current serialized form, followed by the tree data. +pub fn write_shard(writer: &mut W, tree: &PrunableTree) -> io::Result<()> { fn write_inner( mut writer: &mut W, tree: &PrunableTree, @@ -80,6 +81,11 @@ fn read_shard_v1(mut reader: &mut R) -> io::Result(mut reader: R) -> io::Result> { match reader.read_u8()? { SER_V1 => read_shard_v1(&mut reader), @@ -97,7 +103,7 @@ mod tests { use shardtree::testing::arb_prunable_tree; use std::io::Cursor; - use super::{read_shard, write_shard_v1}; + use super::{read_shard, write_shard}; proptest! { #[test] @@ -105,7 +111,7 @@ mod tests { tree in arb_prunable_tree(arb_test_node(), 8, 32) ) { let mut tree_data = vec![]; - write_shard_v1(&mut tree_data, &tree).unwrap(); + write_shard(&mut tree_data, &tree).unwrap(); let cursor = Cursor::new(tree_data); let tree_result = read_shard::(cursor).unwrap(); assert_eq!(tree, tree_result); diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 757fde8bf9..108b5f5d3d 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -640,7 +640,7 @@ pub(crate) fn get_min_unspent_height( /// block, this function does nothing. /// /// This should only be executed inside a transactional context. -pub(crate) fn truncate_to_height( +pub(crate) fn truncate_to_height( conn: &rusqlite::Transaction, params: &P, block_height: BlockHeight, diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 8ce583f9d8..d28b5c7015 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -11,7 +11,7 @@ use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore, TreeS use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer}; -use crate::serialization::{read_shard, write_shard_v1}; +use crate::serialization::{read_shard, write_shard}; pub struct SqliteShardStore { pub(crate) conn: C, @@ -327,7 +327,7 @@ pub(crate) fn put_shard( .map_err(Either::Left)?; let mut subtree_data = vec![]; - write_shard_v1(&mut subtree_data, subtree.root()).map_err(Either::Left)?; + write_shard(&mut subtree_data, subtree.root()).map_err(Either::Left)?; let mut stmt_put_shard = conn .prepare_cached(&format!( @@ -423,7 +423,7 @@ pub(crate) fn put_cap( .map_err(Either::Right)?; let mut cap_data = vec![]; - write_shard_v1(&mut cap_data, &cap).map_err(Either::Left)?; + write_shard(&mut cap_data, &cap).map_err(Either::Left)?; stmt.execute([cap_data]).map_err(Either::Right)?; Ok(()) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 0d292adb12..f4bce16b8a 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -18,10 +18,10 @@ use zcash_primitives::{ sapling, }; -use crate::wallet::{ +use crate::{wallet::{ commitment_tree::SqliteShardStore, init::{migrations::received_notes_nullable_nf, WalletMigrationError}, -}; +}, SAPLING_TABLES_PREFIX}; pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( 0x7da6489d, @@ -94,7 +94,7 @@ impl RusqliteMigration for Migration { let shard_store = SqliteShardStore::<_, sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( transaction, - "sapling", + SAPLING_TABLES_PREFIX, )?; let mut shard_tree: ShardTree< _, @@ -187,14 +187,8 @@ impl RusqliteMigration for Migration { Ok(()) } - fn down(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { - transaction.execute_batch( - "DROP TABLE sapling_tree_checkpoint_marks_removed; - DROP TABLE sapling_tree_checkpoints; - DROP TABLE sapling_tree_cap; - DROP TABLE sapling_tree_shards;", - )?; - - Ok(()) + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + // TODO: something better than just panic? + panic!("Cannot revert this migration."); } } diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 7b32691402..6544b166ec 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -425,7 +425,7 @@ pub(crate) mod tests { }, }; - pub fn test_prover() -> impl TxProver { + pub(crate) fn test_prover() -> impl TxProver { match LocalTxProver::with_default_location() { Some(tx_prover) => tx_prover, None => { diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 47c2f3a4b6..f2404ea94e 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -12,6 +12,9 @@ and this library adheres to Rust's notion of - `Builder::add_orchard_spend` - `Builder::add_orchard_output` - `zcash_primitives::transaction::components::orchard::builder` module +- `impl HashSer for String` is provided under the `test-dependencies` feature + flag. This is a test-only impl; the identity leaf value is `_` and the combining + operation is concatenation. ### Changed - `zcash_primitives::transaction`: From 8fa3a08c0b33c79501bf124714fdbabe542a7b28 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 29 Jun 2023 20:24:33 -0600 Subject: [PATCH 0034/1122] Fix indexing error in checkpoint determination. --- zcash_client_backend/src/welding_rig.rs | 65 +++++++++++++------------ 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index a8a98069ea..22aabb0bb9 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -235,9 +235,9 @@ pub(crate) fn scan_block_with_runner< initial_commitment_tree_meta.map(|m| (m.sapling_tree_size() + 1).into()) }; - for tx in block.vtx.into_iter() { + let block_tx_count = block.vtx.len(); + for (tx_idx, tx) in block.vtx.into_iter().enumerate() { let txid = tx.txid(); - let index = tx.index as usize; // Check for spent notes. The only step that is not constant-time is // the filter() at the end. @@ -338,9 +338,20 @@ pub(crate) fn scan_block_with_runner< .collect() }; - for (index, ((_, output), dec_output)) in decoded.iter().zip(decrypted).enumerate() { + for (output_idx, ((_, output), dec_output)) in decoded.iter().zip(decrypted).enumerate() + { // Collect block note commitments let node = sapling::Node::from_cmu(&output.cmu); + let is_checkpoint = output_idx + 1 == decoded.len() && tx_idx + 1 == block_tx_count; + let retention = match (dec_output.is_some(), is_checkpoint) { + (is_marked, true) => Retention::Checkpoint { + id: block_height, + is_marked, + }, + (true, false) => Retention::Marked, + (false, false) => Retention::Ephemeral, + }; + if let Some((note, account, nk)) = dec_output { // A note is marked as "change" if the account that received it // also spent notes in the same transaction. This will catch, @@ -351,11 +362,11 @@ pub(crate) fn scan_block_with_runner< let is_change = spent_from_accounts.contains(&account); let note_commitment_tree_position = sapling_tree_position .ok_or(SyncError::SaplingTreeSizeUnknown(block_height))? - + index.try_into().unwrap(); + + output_idx.try_into().unwrap(); let nf = K::sapling_nf(&nk, ¬e, note_commitment_tree_position); shielded_outputs.push(WalletSaplingOutput::from_parts( - index, + output_idx, output.cmu, output.ephemeral_key.clone(), account, @@ -364,38 +375,16 @@ pub(crate) fn scan_block_with_runner< note_commitment_tree_position, nf, )); - - sapling_note_commitments.push(( - node, - if index == decoded.len() - 1 { - Retention::Checkpoint { - id: block_height, - is_marked: true, - } - } else { - Retention::Marked - }, - )); - } else { - sapling_note_commitments.push(( - node, - if index == decoded.len() - 1 { - Retention::Checkpoint { - id: block_height, - is_marked: false, - } - } else { - Retention::Ephemeral - }, - )); } + + sapling_note_commitments.push((node, retention)); } } if !(shielded_spends.is_empty() && shielded_outputs.is_empty()) { wtxs.push(WalletTx { txid, - index, + index: tx.index as usize, sapling_spends: shielded_spends, sapling_outputs: shielded_outputs, }); @@ -423,6 +412,7 @@ mod tests { ff::{Field, PrimeField}, GroupEncoding, }; + use incrementalmerkletree::Retention; use rand_core::{OsRng, RngCore}; use zcash_note_encryption::Domain; use zcash_primitives::{ @@ -613,6 +603,21 @@ mod tests { assert_eq!(tx.sapling_outputs[0].index(), 0); assert_eq!(tx.sapling_outputs[0].account(), account); assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); + + assert_eq!( + pruned_block + .sapling_commitments + .iter() + .map(|(_, retention)| *retention) + .collect::>(), + vec![ + Retention::Ephemeral, + Retention::Checkpoint { + id: pruned_block.block_height, + is_marked: true + } + ] + ); } go(false); From c05b3d0c8cc543b3a6780d4df425a9c0e004d8a2 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 08:24:22 -0600 Subject: [PATCH 0035/1122] Add a test demonstrating off-by-one error in `scan_block_with_runner` --- zcash_client_backend/src/welding_rig.rs | 29 +++++++++++++++++-------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index 22aabb0bb9..9a90226515 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -412,7 +412,7 @@ mod tests { ff::{Field, PrimeField}, GroupEncoding, }; - use incrementalmerkletree::Retention; + use incrementalmerkletree::{Retention, Position}; use rand_core::{OsRng, RngCore}; use zcash_note_encryption::Domain; use zcash_primitives::{ @@ -475,13 +475,16 @@ mod tests { /// Create a fake CompactBlock at the given height, with a transaction containing a /// single spend of the given nullifier and a single output paying the given address. /// Returns the CompactBlock. + /// + /// Set `initial_sapling_tree_size` to `None` to simulate a `CompactBlock` retrieved + /// from a `lightwalletd` that is not currently tracking note commitment tree sizes. fn fake_compact_block( height: BlockHeight, nf: Nullifier, dfvk: &DiversifiableFullViewingKey, value: Amount, tx_after: bool, - initial_sapling_tree_size: u32, + initial_sapling_tree_size: Option, ) -> CompactBlock { let to = dfvk.default_address().1; @@ -541,9 +544,12 @@ mod tests { cb.vtx.push(tx); } - cb.block_metadata = Some(BlockMetadata { - sapling_commitment_tree_size: initial_sapling_tree_size - + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), + cb.block_metadata = initial_sapling_tree_size.map(|s| BlockMetadata { + sapling_commitment_tree_size: s + cb + .vtx + .iter() + .map(|tx| tx.outputs.len() as u32) + .sum::(), ..Default::default() }); @@ -563,7 +569,7 @@ mod tests { &dfvk, Amount::from_u64(5).unwrap(), false, - 0, + None, ); assert_eq!(cb.vtx.len(), 2); @@ -603,7 +609,12 @@ mod tests { assert_eq!(tx.sapling_outputs[0].index(), 0); assert_eq!(tx.sapling_outputs[0].account(), account); assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); + assert_eq!( + tx.sapling_outputs[0].note_commitment_tree_position(), + Position::from(1) + ); + assert_eq!(pruned_block.sapling_commitment_tree_size, Some(2)); assert_eq!( pruned_block .sapling_commitments @@ -637,7 +648,7 @@ mod tests { &dfvk, Amount::from_u64(5).unwrap(), true, - 0, + Some(0), ); assert_eq!(cb.vtx.len(), 3); @@ -663,7 +674,7 @@ mod tests { cb, &[(&AccountId::from(0), &dfvk)], &[], - Some(&CommitmentTreeMeta::from_parts(0)), + None, batch_runner.as_mut(), ) .unwrap(); @@ -712,7 +723,7 @@ mod tests { &dfvk, Amount::from_u64(5).unwrap(), false, - 0, + Some(0), ); assert_eq!(cb.vtx.len(), 2); let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; From 45177a51e1836bf228ffdff5ec01cbf02b9707ca Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 09:26:00 -0600 Subject: [PATCH 0036/1122] Fix off-by-one error in scan_block_with_runner. --- zcash_client_backend/src/data_api/chain.rs | 8 ++-- zcash_client_backend/src/welding_rig.rs | 47 +++++++++++----------- zcash_client_sqlite/src/wallet.rs | 2 +- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 66355b06aa..b09432198a 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -103,19 +103,19 @@ use error::{ChainError, Error}; /// Metadata describing the sizes of the zcash note commitment trees as of a particular block. pub struct CommitmentTreeMeta { - sapling_tree_size: u64, - //TODO: orchard_tree_size: u64 + sapling_tree_size: u32, + //TODO: orchard_tree_size: u32 } impl CommitmentTreeMeta { /// Constructs a new [`CommitmentTreeMeta`] value from its constituent parts. - pub fn from_parts(sapling_tree_size: u64) -> Self { + pub fn from_parts(sapling_tree_size: u32) -> Self { Self { sapling_tree_size } } /// Returns the size of the Sapling note commitment tree as of the block that this /// [`CommitmentTreeMeta`] describes. - pub fn sapling_tree_size(&self) -> u64 { + pub fn sapling_tree_size(&self) -> u32 { self.sapling_tree_size } } diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index 9a90226515..79e3a4c394 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -218,22 +218,24 @@ pub(crate) fn scan_block_with_runner< // to use it. `block.sapling_commitment_tree_size` is expected to be correct as of the end of // the block, and we can't have a note of ours in a block with no outputs so treating the zero // default value from the protobuf as `None` is always correct. - let mut sapling_tree_position = if let Some(sapling_tree_size) = block + let mut sapling_commitment_tree_size = block .block_metadata .as_ref() - .map(|m| m.sapling_commitment_tree_size) - .filter(|s| *s != 0) - { - let end_position_exclusive = Position::from(u64::from(sapling_tree_size)); - let output_count = block - .vtx - .iter() - .map(|tx| u64::try_from(tx.outputs.len()).unwrap()) - .sum(); - Some(end_position_exclusive - output_count) - } else { - initial_commitment_tree_meta.map(|m| (m.sapling_tree_size() + 1).into()) - }; + .and_then(|m| { + if m.sapling_commitment_tree_size == 0 { + None + } else { + let block_note_count: u32 = block + .vtx + .iter() + .map(|tx| { + u32::try_from(tx.outputs.len()).expect("output count cannot exceed a u32") + }) + .sum(); + Some(m.sapling_commitment_tree_size - block_note_count) + } + }) + .or_else(|| initial_commitment_tree_meta.map(|m| m.sapling_tree_size())); let block_tx_count = block.vtx.len(); for (tx_idx, tx) in block.vtx.into_iter().enumerate() { @@ -274,7 +276,7 @@ pub(crate) fn scan_block_with_runner< // Check for incoming notes while incrementing tree and witnesses let mut shielded_outputs: Vec> = vec![]; - let tx_outputs_len = u64::try_from(tx.outputs.len()).unwrap(); + let tx_outputs_len = u32::try_from(tx.outputs.len()).unwrap(); { let decoded = &tx .outputs @@ -360,9 +362,9 @@ pub(crate) fn scan_block_with_runner< // - Notes created by consolidation transactions. // - Notes sent from one account to itself. let is_change = spent_from_accounts.contains(&account); - let note_commitment_tree_position = sapling_tree_position - .ok_or(SyncError::SaplingTreeSizeUnknown(block_height))? - + output_idx.try_into().unwrap(); + let note_commitment_tree_position = sapling_commitment_tree_size + .map(|s| Position::from(u64::from(s + u32::try_from(output_idx).unwrap()))) + .ok_or(SyncError::SaplingTreeSizeUnknown(block_height))?; let nf = K::sapling_nf(&nk, ¬e, note_commitment_tree_position); shielded_outputs.push(WalletSaplingOutput::from_parts( @@ -390,7 +392,7 @@ pub(crate) fn scan_block_with_runner< }); } - sapling_tree_position = sapling_tree_position.map(|pos| pos + tx_outputs_len); + sapling_commitment_tree_size = sapling_commitment_tree_size.map(|s| s + tx_outputs_len); } Ok(PrunedBlock { @@ -398,10 +400,7 @@ pub(crate) fn scan_block_with_runner< block_hash, block_time: block.time, transactions: wtxs, - sapling_commitment_tree_size: block - .block_metadata - .map(|m| m.sapling_commitment_tree_size) - .filter(|s| *s != 0), + sapling_commitment_tree_size, sapling_commitments: sapling_note_commitments, }) } @@ -412,7 +411,7 @@ mod tests { ff::{Field, PrimeField}, GroupEncoding, }; - use incrementalmerkletree::{Retention, Position}; + use incrementalmerkletree::{Position, Retention}; use rand_core::{OsRng, RngCore}; use zcash_note_encryption::Domain; use zcash_primitives::{ diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 108b5f5d3d..cbce42617b 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -551,7 +551,7 @@ pub(crate) fn fully_scanned_height( [], |row| { let max_height: u32 = row.get(0)?; - let sapling_tree_size: Option = row.get(1)?; + let sapling_tree_size: Option = row.get(1)?; let sapling_tree: Vec = row.get(2)?; Ok(( BlockHeight::from(max_height), From 95745dd6207d93059b343088d486319e6eb9f628 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 09:45:06 -0600 Subject: [PATCH 0037/1122] Use ruqlite::Rows::mapped to allow `collect` --- .../src/wallet/commitment_tree.rs | 43 +++++++------------ .../init/migrations/shardtree_support.rs | 11 +++-- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index d28b5c7015..885195767f 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -542,7 +542,6 @@ pub(crate) fn get_checkpoint( checkpoint_position .map(|pos_opt| { - let mut marks_removed = BTreeSet::new(); let mut stmt = conn .prepare_cached(&format!( "SELECT mark_removed_position @@ -551,17 +550,14 @@ pub(crate) fn get_checkpoint( table_prefix )) .map_err(Either::Right)?; - let mut mark_removed_rows = stmt + let mark_removed_rows = stmt .query([u32::from(checkpoint_id)]) .map_err(Either::Right)?; - while let Some(row) = mark_removed_rows.next().map_err(Either::Right)? { - marks_removed.insert( - row.get::<_, u64>(0) - .map(Position::from) - .map_err(Either::Right)?, - ); - } + let marks_removed = mark_removed_rows + .mapped(|row| row.get::<_, u64>(0).map(Position::from)) + .collect::, _>>() + .map_err(Either::Right)?; Ok(Checkpoint::from_parts( pos_opt.map_or(TreeState::Empty, TreeState::AtPosition), @@ -605,7 +601,6 @@ pub(crate) fn get_checkpoint_at_depth( checkpoint_parts .map(|(checkpoint_id, pos_opt)| { - let mut marks_removed = BTreeSet::new(); let mut stmt = conn .prepare_cached(&format!( "SELECT mark_removed_position @@ -614,17 +609,14 @@ pub(crate) fn get_checkpoint_at_depth( table_prefix )) .map_err(Either::Right)?; - let mut mark_removed_rows = stmt + let mark_removed_rows = stmt .query([u32::from(checkpoint_id)]) .map_err(Either::Right)?; - while let Some(row) = mark_removed_rows.next().map_err(Either::Right)? { - marks_removed.insert( - row.get::<_, u64>(0) - .map(Position::from) - .map_err(Either::Right)?, - ); - } + let marks_removed = mark_removed_rows + .mapped(|row| row.get::<_, u64>(0).map(Position::from)) + .collect::, _>>() + .map_err(Either::Right)?; Ok(( checkpoint_id, @@ -675,17 +667,14 @@ where .map(|opt| opt.map_or_else(|| TreeState::Empty, |p| TreeState::AtPosition(p.into()))) .map_err(Either::Right)?; - let mut mark_removed_rows = stmt_get_checkpoint_marks_removed + let mark_removed_rows = stmt_get_checkpoint_marks_removed .query(named_params![":checkpoint_id": checkpoint_id]) .map_err(Either::Right)?; - let mut marks_removed = BTreeSet::new(); - while let Some(mr_row) = mark_removed_rows.next().map_err(Either::Right)? { - let mark_removed_position = mr_row - .get::<_, u64>(0) - .map(Position::from) - .map_err(Either::Right)?; - marks_removed.insert(mark_removed_position); - } + + let marks_removed = mark_removed_rows + .mapped(|row| row.get::<_, u64>(0).map(Position::from)) + .collect::, _>>() + .map_err(Either::Right)?; callback( &BlockHeight::from(checkpoint_id), diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index f4bce16b8a..08708d65cb 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -18,10 +18,13 @@ use zcash_primitives::{ sapling, }; -use crate::{wallet::{ - commitment_tree::SqliteShardStore, - init::{migrations::received_notes_nullable_nf, WalletMigrationError}, -}, SAPLING_TABLES_PREFIX}; +use crate::{ + wallet::{ + commitment_tree::SqliteShardStore, + init::{migrations::received_notes_nullable_nf, WalletMigrationError}, + }, + SAPLING_TABLES_PREFIX, +}; pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( 0x7da6489d, From cd939f94c42a2726fe47ddccef8dea80b4ef1f84 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 09:48:19 -0600 Subject: [PATCH 0038/1122] Ensure that checkpoints are ordered by position when querying for pruning. --- zcash_client_sqlite/src/wallet/commitment_tree.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 885195767f..78548836aa 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -642,6 +642,7 @@ where .prepare_cached(&format!( "SELECT checkpoint_id, position FROM {}_tree_checkpoints + ORDER BY position LIMIT :limit", table_prefix )) From 70497a241cb0382758db4d9f3dfae209f393a999 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 10:42:48 -0600 Subject: [PATCH 0039/1122] Only store z->t transaction data once, not once per Sapling output. --- zcash_client_sqlite/src/lib.rs | 83 +++++++++++++++++----------------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 9c1ac261a7..01c613c2da 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -447,50 +447,51 @@ impl WalletWrite for WalletDb d_tx: DecryptedTransaction, ) -> Result { self.transactionally(|wdb| { - let tx_ref = wallet::put_tx_data(wdb.conn.0, d_tx.tx, None, None)?; - - let mut spending_account_id: Option = None; - for output in d_tx.sapling_outputs { - match output.transfer_type { - TransferType::Outgoing | TransferType::WalletInternal => { - let recipient = if output.transfer_type == TransferType::Outgoing { - Recipient::Sapling(output.note.recipient()) - } else { - Recipient::InternalAccount(output.account, PoolType::Sapling) - }; - - wallet::put_sent_output( - wdb.conn.0, - &wdb.params, - output.account, - tx_ref, - output.index, - &recipient, - Amount::from_u64(output.note.value().inner()).map_err(|_| { - SqliteClientError::CorruptedData( - "Note value is not a valid Zcash amount.".to_string(), - ) - })?, - Some(&output.memo), - )?; + let tx_ref = wallet::put_tx_data(wdb.conn.0, d_tx.tx, None, None)?; + + let mut spending_account_id: Option = None; + for output in d_tx.sapling_outputs { + match output.transfer_type { + TransferType::Outgoing | TransferType::WalletInternal => { + let recipient = if output.transfer_type == TransferType::Outgoing { + Recipient::Sapling(output.note.recipient()) + } else { + Recipient::InternalAccount(output.account, PoolType::Sapling) + }; - if matches!(recipient, Recipient::InternalAccount(_, _)) { - wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; + wallet::put_sent_output( + wdb.conn.0, + &wdb.params, + output.account, + tx_ref, + output.index, + &recipient, + Amount::from_u64(output.note.value().inner()).map_err(|_| { + SqliteClientError::CorruptedData( + "Note value is not a valid Zcash amount.".to_string(), + ) + })?, + Some(&output.memo), + )?; + + if matches!(recipient, Recipient::InternalAccount(_, _)) { + wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; + } } - } - TransferType::Incoming => { - match spending_account_id { - Some(id) => { - if id != output.account { - panic!("Unable to determine a unique account identifier for z->t spend."); + TransferType::Incoming => { + match spending_account_id { + Some(id) => { + if id != output.account { + panic!("Unable to determine a unique account identifier for z->t spend."); + } + } + None => { + spending_account_id = Some(output.account); } } - None => { - spending_account_id = Some(output.account); - } - } - wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; + wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; + } } } @@ -527,8 +528,8 @@ impl WalletWrite for WalletDb } } } - } - Ok(tx_ref) + + Ok(tx_ref) }) } From 8625e9a7771752340a312d6365daae000c40dfbb Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 12:05:15 -0600 Subject: [PATCH 0040/1122] Handle parsing of the not-present `CommitmentTree` sentinel. --- zcash_client_sqlite/src/wallet.rs | 42 ++++++++++++++++++------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index cbce42617b..7847c880bf 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -108,6 +108,8 @@ pub(crate) mod commitment_tree; pub mod init; pub(crate) mod sapling; +pub(crate) const BLOCK_SAPLING_FRONTIER_ABSENT: &[u8] = &[0x0]; + pub(crate) fn pool_code(pool_type: PoolType) -> i64 { // These constants are *incidentally* shared with the typecodes // for unified addresses, but this is exclusively an internal @@ -563,23 +565,29 @@ pub(crate) fn fully_scanned_height( .optional()?; res_opt - .map(|(max_height, sapling_tree_size, sapling_tree)| { - let commitment_tree_meta = - CommitmentTreeMeta::from_parts(if let Some(known_size) = sapling_tree_size { - known_size - } else { - // parse the legacy commitment tree data - read_commitment_tree::< - zcash_primitives::sapling::Node, - _, - { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, - >(Cursor::new(sapling_tree))? - .size() - .try_into() - .expect("usize values are convertible to u64 on all supported platforms.") - }); - - Ok((max_height, commitment_tree_meta)) + .and_then(|(max_height, sapling_tree_size, sapling_tree)| { + sapling_tree_size + .map(|s| Ok(CommitmentTreeMeta::from_parts(s))) + .or_else(|| { + if &sapling_tree == BLOCK_SAPLING_FRONTIER_ABSENT { + None + } else { + // parse the legacy commitment tree data + read_commitment_tree::< + zcash_primitives::sapling::Node, + _, + { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, + >(Cursor::new(sapling_tree)) + .map(|tree| { + Some(CommitmentTreeMeta::from_parts( + tree.size().try_into().unwrap(), + )) + }) + .map_err(SqliteClientError::from) + .transpose() + } + }) + .map(|meta_res| meta_res.map(|meta| (max_height, meta))) }) .transpose() } From e225a54d2e882b7a8ed54e894797ab786adab28f Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 12:37:41 -0600 Subject: [PATCH 0041/1122] Use `NonZeroU32` for all `min_confirmations` values. --- zcash_client_backend/CHANGELOG.md | 6 ++ zcash_client_backend/src/data_api.rs | 5 +- zcash_client_backend/src/data_api/wallet.rs | 45 ++++---------- .../src/data_api/wallet/input_selection.rs | 9 +-- zcash_client_sqlite/src/chain.rs | 4 +- zcash_client_sqlite/src/wallet.rs | 11 +++- zcash_client_sqlite/src/wallet/sapling.rs | 59 ++++++++++++------- 7 files changed, 78 insertions(+), 61 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 0110fa88b1..fa7d27cc99 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -30,6 +30,7 @@ and this library adheres to Rust's notion of wallet states where the full note plaintext is not available. - `WalletRead::get_nullifiers` has been renamed to `WalletRead::get_sapling_nullifiers` and its signature has changed; it now subsumes the removed `WalletRead::get_all_nullifiers`. + - `WalletRead::get_target_and_anchor_heights` now takes its argument as a `NonZeroU32` - `chain::scan_cached_blocks` now takes a `from_height` argument that permits the caller to control the starting position of the scan range. - A new `CommitmentTree` variant has been added to `data_api::error::Error` @@ -38,6 +39,11 @@ and this library adheres to Rust's notion of implemented for the type passed to them for the `wallet_db` parameter. - `data_api::wallet::create_proposed_transaction` now takes an additional `min_confirmations` argument. + - `data_api::wallet::{spend, create_spend_to_address, shield_transparent_funds, + propose_transfer, propose_shielding, create_proposed_transaction}` now take their + respective `min_confirmations` arguments as `NonZeroU32` + - `data_api::wallet::input_selection::InputSelector::{propose_transaction, propose_shielding}` + now take their respective `min_confirmations` arguments as `NonZeroU32` - A new `Sync` variant has been added to `data_api::chain::error::Error`. - A new `SyncRequired` variant has been added to `data_api::wallet::input_selection::InputSelectorError`. - `zcash_client_backend::wallet`: diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index d1031e5701..5b199b422f 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::fmt::Debug; +use std::num::NonZeroU32; use std::{cmp, ops::Range}; use incrementalmerkletree::Retention; @@ -97,7 +98,7 @@ pub trait WalletRead { /// This will return `Ok(None)` if no block data is present in the database. fn get_target_and_anchor_heights( &self, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result, Self::Error> { self.block_height_extrema().map(|heights| { heights.map(|(min_height, max_height)| { @@ -106,7 +107,7 @@ pub trait WalletRead { // Select an anchor min_confirmations back from the target block, // unless that would be before the earliest block we have. let anchor_height = BlockHeight::from(cmp::max( - u32::from(target_height).saturating_sub(min_confirmations), + u32::from(target_height).saturating_sub(min_confirmations.into()), u32::from(min_height), )); diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index a475289209..c423c02eac 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1,4 +1,4 @@ -use std::convert::Infallible; +use std::{convert::Infallible, num::NonZeroU32}; use std::fmt::Debug; use shardtree::{ShardStore, ShardTree, ShardTreeError}; @@ -122,10 +122,6 @@ where /// spent. A value of 10 confirmations is recommended and 0-conf transactions are /// not supported. /// -/// # Panics -/// -/// Panics if `min_confirmations == 0`; 0-conf transactions are not supported. -/// /// # Examples /// /// ``` @@ -203,7 +199,7 @@ pub fn create_spend_to_address( amount: Amount, memo: Option, ovk_policy: OvkPolicy, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result< DbT::TxRef, Error< @@ -292,7 +288,8 @@ where /// can allow the sender to view the resulting notes on the blockchain. /// * `min_confirmations`: The minimum number of confirmations that a previously /// received note must have in the blockchain in order to be considered for being -/// spent. A value of 10 confirmations is recommended. +/// spent. A value of 10 confirmations is recommended and 0-conf transactions are +/// not supported. /// /// [`sapling::TxProver`]: zcash_primitives::sapling::prover::TxProver #[allow(clippy::too_many_arguments)] @@ -305,7 +302,7 @@ pub fn spend( usk: &UnifiedSpendingKey, request: zip321::TransactionRequest, ovk_policy: OvkPolicy, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result< DbT::TxRef, Error< @@ -323,10 +320,6 @@ where ParamsT: consensus::Parameters + Clone, InputsT: InputSelector, { - assert!( - min_confirmations > 0, - "zero-conf transactions are not supported" - ); let account = wallet_db .get_account_for_ufvk(&usk.to_unified_full_viewing_key()) .map_err(Error::DataSource)? @@ -364,7 +357,7 @@ pub fn propose_transfer( spend_from_account: AccountId, input_selector: &InputsT, request: zip321::TransactionRequest, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result< Proposal, Error< @@ -381,10 +374,6 @@ where ParamsT: consensus::Parameters + Clone, InputsT: InputSelector, { - assert!( - min_confirmations > 0, - "zero-conf transactions are not supported" - ); input_selector .propose_transaction( params, @@ -405,7 +394,7 @@ pub fn propose_shielding( input_selector: &InputsT, shielding_threshold: NonNegativeAmount, from_addrs: &[TransparentAddress], - min_confirmations: u32, + min_confirmations: NonZeroU32 ) -> Result< Proposal, Error< @@ -422,10 +411,6 @@ where DbT::NoteRef: Copy + Eq + Ord, InputsT: InputSelector, { - assert!( - min_confirmations > 0, - "zero-conf transactions are not supported" - ); input_selector .propose_shielding( params, @@ -451,7 +436,7 @@ pub fn create_proposed_transaction( usk: &UnifiedSpendingKey, ovk_policy: OvkPolicy, proposal: Proposal, - min_confirmations: u32, + min_confirmations: NonZeroU32, change_memo: Option, ) -> Result< DbT::TxRef, @@ -470,10 +455,6 @@ where ParamsT: consensus::Parameters + Clone, FeeRuleT: FeeRule, { - assert!( - min_confirmations > 0, - "zero-conf transactions are not supported" - ); let account = wallet_db .get_account_for_ufvk(&usk.to_unified_full_viewing_key()) .map_err(Error::DataSource)? @@ -516,8 +497,7 @@ where selected, usk.sapling(), &dfvk, - usize::try_from(min_confirmations - 1) - .expect("min_confirmations should never be anywhere close to usize::MAX"), + usize::try_from(u32::from(min_confirmations) - 1).unwrap() )? .ok_or(Error::NoteMismatch(selected.note_id))?; @@ -711,8 +691,9 @@ where /// to the wallet that the wallet can use to improve how it represents those /// shielding transactions to the user. /// * `min_confirmations`: The minimum number of confirmations that a previously -/// received UTXO must have in the blockchain in order to be considered for being -/// spent. +/// received note must have in the blockchain in order to be considered for being +/// spent. A value of 10 confirmations is recommended and 0-conf transactions are +/// not supported. /// /// [`sapling::TxProver`]: zcash_primitives::sapling::prover::TxProver #[cfg(feature = "transparent-inputs")] @@ -727,7 +708,7 @@ pub fn shield_transparent_funds( usk: &UnifiedSpendingKey, from_addrs: &[TransparentAddress], memo: &MemoBytes, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result< DbT::TxRef, Error< diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 90c7013360..578082a337 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -2,6 +2,7 @@ use core::marker::PhantomData; use std::fmt; +use std::num::NonZeroU32; use std::{collections::BTreeSet, fmt::Debug}; use zcash_primitives::{ @@ -180,7 +181,7 @@ pub trait InputSelector { wallet_db: &Self::DataSource, account: AccountId, transaction_request: TransactionRequest, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result< Proposal::DataSource as WalletRead>::NoteRef>, InputSelectorError<<::DataSource as WalletRead>::Error, Self::Error>, @@ -204,7 +205,7 @@ pub trait InputSelector { wallet_db: &Self::DataSource, shielding_threshold: NonNegativeAmount, source_addrs: &[TransparentAddress], - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result< Proposal::DataSource as WalletRead>::NoteRef>, InputSelectorError<<::DataSource as WalletRead>::Error, Self::Error>, @@ -324,7 +325,7 @@ where wallet_db: &Self::DataSource, account: AccountId, transaction_request: TransactionRequest, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result, InputSelectorError> where ParamsT: consensus::Parameters, @@ -442,7 +443,7 @@ where wallet_db: &Self::DataSource, shielding_threshold: NonNegativeAmount, source_addrs: &[TransparentAddress], - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result, InputSelectorError> where ParamsT: consensus::Parameters, diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index f5300de90c..191b2a6b0a 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -265,6 +265,8 @@ where #[cfg(test)] #[allow(deprecated)] mod tests { + use std::num::NonZeroU32; + use secrecy::Secret; use tempfile::NamedTempFile; @@ -681,7 +683,7 @@ mod tests { &usk, req, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Ok(_) ); diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 7847c880bf..722153dbf5 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -544,6 +544,8 @@ pub(crate) fn block_height_extrema( pub(crate) fn fully_scanned_height( conn: &rusqlite::Connection, ) -> Result, SqliteClientError> { + // FIXME: this will need to be rewritten once out-of-order scan range suggestion + // is implemented. let res_opt = conn .query_row( "SELECT height, sapling_commitment_tree_size, sapling_tree @@ -1155,6 +1157,8 @@ pub(crate) fn put_sent_output( #[cfg(test)] mod tests { + use std::num::NonZeroU32; + use secrecy::Secret; use tempfile::NamedTempFile; @@ -1197,7 +1201,12 @@ mod tests { ); // We can't get an anchor height, as we have not scanned any blocks. - assert_eq!(db_data.get_target_and_anchor_heights(10).unwrap(), None); + assert_eq!( + db_data + .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) + .unwrap(), + None + ); // An invalid account has zero balance assert_matches!( diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 6544b166ec..85e101c738 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -367,6 +367,8 @@ pub(crate) fn put_received_note( #[cfg(test)] #[allow(deprecated)] pub(crate) mod tests { + use std::num::NonZeroU32; + use rusqlite::Connection; use secrecy::Secret; use tempfile::NamedTempFile; @@ -461,7 +463,7 @@ pub(crate) mod tests { Amount::from_u64(1).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Err(data_api::error::Error::KeyNotRecognized) ); @@ -490,7 +492,7 @@ pub(crate) mod tests { Amount::from_u64(1).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Err(data_api::error::Error::ScanRequired) ); @@ -533,7 +535,7 @@ pub(crate) mod tests { Amount::from_u64(1).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Err(data_api::error::Error::InsufficientFunds { available, @@ -572,7 +574,10 @@ pub(crate) mod tests { scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance - let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); + let (_, anchor_height) = db_data + .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) + .unwrap() + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -595,7 +600,10 @@ pub(crate) mod tests { scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance does not include the second note - let (_, anchor_height2) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); + let (_, anchor_height2) = db_data + .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) + .unwrap() + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), (value + value).unwrap() @@ -618,7 +626,7 @@ pub(crate) mod tests { Amount::from_u64(70000).unwrap(), None, OvkPolicy::Sender, - 10, + NonZeroU32::new(10).unwrap(), ), Err(data_api::error::Error::InsufficientFunds { available, @@ -654,7 +662,7 @@ pub(crate) mod tests { Amount::from_u64(70000).unwrap(), None, OvkPolicy::Sender, - 10, + NonZeroU32::new(10).unwrap(), ), Err(data_api::error::Error::InsufficientFunds { available, @@ -687,7 +695,7 @@ pub(crate) mod tests { Amount::from_u64(70000).unwrap(), None, OvkPolicy::Sender, - 10, + NonZeroU32::new(10).unwrap(), ), Ok(_) ); @@ -738,7 +746,7 @@ pub(crate) mod tests { Amount::from_u64(15000).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Ok(_) ); @@ -754,7 +762,7 @@ pub(crate) mod tests { Amount::from_u64(2000).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Err(data_api::error::Error::InsufficientFunds { available, @@ -789,7 +797,7 @@ pub(crate) mod tests { Amount::from_u64(2000).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Err(data_api::error::Error::InsufficientFunds { available, @@ -820,7 +828,7 @@ pub(crate) mod tests { Amount::from_u64(2000).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ) .unwrap(); } @@ -872,7 +880,7 @@ pub(crate) mod tests { Amount::from_u64(15000).unwrap(), None, ovk_policy, - 1, + NonZeroU32::new(1).unwrap(), ) .unwrap(); @@ -960,7 +968,10 @@ pub(crate) mod tests { scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance - let (_, anchor_height) = db_data.get_target_and_anchor_heights(1).unwrap().unwrap(); + let (_, anchor_height) = db_data + .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) + .unwrap() + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -981,7 +992,7 @@ pub(crate) mod tests { Amount::from_u64(50000).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Ok(_) ); @@ -1016,7 +1027,10 @@ pub(crate) mod tests { scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance - let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); + let (_, anchor_height) = db_data + .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) + .unwrap() + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -1037,7 +1051,7 @@ pub(crate) mod tests { Amount::from_u64(50000).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Ok(_) ); @@ -1086,7 +1100,10 @@ pub(crate) mod tests { // Verified balance matches total balance let total = Amount::from_u64(60000).unwrap(); - let (_, anchor_height) = db_data.get_target_and_anchor_heights(1).unwrap().unwrap(); + let (_, anchor_height) = db_data + .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) + .unwrap() + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), total @@ -1121,7 +1138,7 @@ pub(crate) mod tests { &usk, req, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Err(Error::InsufficientFunds { available, required }) if available == Amount::from_u64(51000).unwrap() @@ -1149,7 +1166,7 @@ pub(crate) mod tests { &usk, req, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Ok(_) ); @@ -1213,7 +1230,7 @@ pub(crate) mod tests { &usk, &[*taddr], &MemoBytes::empty(), - 1 + NonZeroU32::new(1).unwrap() ), Ok(_) ); From 77b638012b3704082364377f00f87aea1e45c74d Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Sat, 1 Jul 2023 17:58:01 -0600 Subject: [PATCH 0042/1122] Remove `zcash_client_backend::data_api::chain::validate_chain` Local chain validation will be performed internal to `scan_cached_blocks`, and as handling of chain reorgs will need to change to support out-of-order scanning, the `validate_chain` method will be superfluous. It is removed in advance of other changes in order to avoid updating it to reflect the forthcoming changes. --- zcash_client_backend/CHANGELOG.md | 2 + zcash_client_backend/src/data_api/chain.rs | 101 +-------------------- zcash_client_sqlite/src/chain.rs | 40 ++------ 3 files changed, 16 insertions(+), 127 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index fa7d27cc99..8a6fc7f364 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -75,6 +75,8 @@ and this library adheres to Rust's notion of feature flag, has been modified by the addition of a `sapling_tree` property. - `wallet::input_selection`: - `Proposal::target_height` (use `Proposal::min_target_height` instead). +- `zcash_client_backend::data_api::chain::validate_chain` TODO: document how + to handle validation given out-of-order blocks. - `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}` have been removed as individual incremental witnesses are no longer tracked on a per-note basis. The global note commitment tree for the wallet should be used diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index b09432198a..a554703077 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -36,43 +36,12 @@ //! let mut db_data = testing::MockWalletDb::new(Network::TestNetwork); //! //! // 1) Download new CompactBlocks into block_source. -//! -//! // 2) Run the chain validator on the received blocks. //! // -//! // Given that we assume the server always gives us correct-at-the-time blocks, any -//! // errors are in the blocks we have previously cached or scanned. -//! let max_height_hash = db_data.get_max_height_hash().map_err(Error::Wallet)?; -//! if let Err(e) = validate_chain(&block_source, max_height_hash, None) { -//! match e { -//! Error::Chain(e) => { -//! // a) Pick a height to rewind to. -//! // -//! // This might be informed by some external chain reorg information, or -//! // heuristics such as the platform, available bandwidth, size of recent -//! // CompactBlocks, etc. -//! let rewind_height = e.at_height() - 10; -//! -//! // b) Rewind scanned block information. -//! db_data.truncate_to_height(rewind_height); -//! -//! // c) Delete cached blocks from rewind_height onwards. -//! // -//! // This does imply that assumed-valid blocks will be re-downloaded, but it -//! // is also possible that in the intervening time, a chain reorg has -//! // occurred that orphaned some of those blocks. -//! -//! // d) If there is some separate thread or service downloading -//! // CompactBlocks, tell it to go back and download from rewind_height -//! // onwards. -//! }, -//! e => { -//! // handle or return other errors -//! -//! } -//! } -//! } -//! -//! // 3) Scan (any remaining) cached blocks. +//! // 2) FIXME: Obtain necessary block metadata for continuity checking? +//! // +//! // 3) Scan cached blocks. +//! // +//! // FIXME: update documentation on how to detect when a rewind is required. //! // //! // At this point, the cache and scanned data are locally consistent (though not //! // necessarily consistent with the latest chain tip - this would be discovered the @@ -82,10 +51,7 @@ //! # } //! ``` -use std::convert::Infallible; - use zcash_primitives::{ - block::BlockHash, consensus::{self, BlockHeight}, sapling::{self, note_encryption::PreparedIncomingViewingKey}, zip32::Scope, @@ -143,63 +109,6 @@ pub trait BlockSource { F: FnMut(CompactBlock) -> Result<(), error::Error>; } -/// Checks that the scanned blocks in the data database, when combined with the recent -/// `CompactBlock`s in the block_source database, form a valid chain. -/// -/// This function is built on the core assumption that the information provided in the -/// block source is more likely to be accurate than the previously-scanned information. -/// This follows from the design (and trust) assumption that the `lightwalletd` server -/// provides accurate block information as of the time it was requested. -/// -/// Arguments: -/// - `block_source` Source of compact blocks -/// - `validate_from` Height & hash of last validated block; -/// - `limit` specified number of blocks that will be valididated. Callers providing -/// a `limit` argument are responsible of making subsequent calls to `validate_chain()` -/// to complete validating the remaining blocks stored on the `block_source`. If `none` -/// is provided, there will be no limit set to the validation and upper bound of the -/// validation range will be the latest height present in the `block_source`. -/// -/// Returns: -/// - `Ok(())` if the combined chain is valid up to the given height -/// and block hash. -/// - `Err(Error::Chain(cause))` if the combined chain is invalid. -/// - `Err(e)` if there was an error during validation unrelated to chain validity. -pub fn validate_chain( - block_source: &BlockSourceT, - mut validate_from: Option<(BlockHeight, BlockHash)>, - limit: Option, -) -> Result<(), Error> -where - BlockSourceT: BlockSource, -{ - // The block source will contain blocks above the `validate_from` height. Validate from that - // maximum height up to the chain tip, returning the hash of the block found in the block - // source at the `validate_from` height, which can then be used to verify chain integrity by - // comparing against the `validate_from` hash. - - block_source.with_blocks::<_, Infallible, Infallible>( - validate_from.map(|(h, _)| h + 1), - limit, - move |block| { - if let Some((valid_height, valid_hash)) = validate_from { - if block.height() != valid_height + 1 { - return Err(ChainError::block_height_discontinuity( - valid_height + 1, - block.height(), - ) - .into()); - } else if block.prev_hash() != valid_hash { - return Err(ChainError::prev_hash_mismatch(block.height()).into()); - } - } - - validate_from = Some((block.height(), block.hash())); - Ok(()) - }, - ) -} - /// Scans at most `limit` new blocks added to the block source for any transactions received by the /// tracked accounts. /// diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 191b2a6b0a..4d968bb360 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -279,7 +279,7 @@ mod tests { use zcash_client_backend::{ address::RecipientAddress, data_api::{ - chain::{error::Error, scan_cached_blocks, validate_chain}, + chain::scan_cached_blocks, wallet::{input_selection::GreedyInputSelector, spend}, WalletRead, WalletWrite, }, @@ -329,21 +329,9 @@ mod tests { insert_into_cache(&db_cache, &cb); - // Cache-only chain should be valid - let validate_chain_result = validate_chain( - &db_cache, - Some((fake_block_height, fake_block_hash)), - Some(1), - ); - - assert_matches!(validate_chain_result, Ok(())); - // Scan the cache scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); - // Data-only chain should be valid - validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); - // Create a second fake CompactBlock sending more value to the address let (cb2, _) = fake_compact_block( sapling_activation_height() + 1, @@ -355,14 +343,8 @@ mod tests { ); insert_into_cache(&db_cache, &cb2); - // Data+cache chain should be valid - validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); - // Scan the cache again scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); - - // Data-only chain should be valid - validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); } #[test] @@ -401,9 +383,6 @@ mod tests { // Scan the cache scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); - // Data-only chain should be valid - validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); - // Create more fake CompactBlocks that don't connect to the scanned ones let (cb3, _) = fake_compact_block( sapling_activation_height() + 2, @@ -425,9 +404,10 @@ mod tests { insert_into_cache(&db_cache, &cb4); // Data+cache chain should be invalid at the data/cache boundary - let val_result = validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None); - - assert_matches!(val_result, Err(Error::Chain(e)) if e.at_height() == sapling_activation_height() + 2); + assert_matches!( + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None), + Err(_) // FIXME: check error result more closely + ); } #[test] @@ -466,9 +446,6 @@ mod tests { // Scan the cache scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); - // Data-only chain should be valid - validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); - // Create more fake CompactBlocks that contain a reorg let (cb3, _) = fake_compact_block( sapling_activation_height() + 2, @@ -490,9 +467,10 @@ mod tests { insert_into_cache(&db_cache, &cb4); // Data+cache chain should be invalid inside the cache - let val_result = validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None); - - assert_matches!(val_result, Err(Error::Chain(e)) if e.at_height() == sapling_activation_height() + 3); + assert_matches!( + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None), + Err(_) // FIXME: check error result more closely + ); } #[test] From e3aafdad19d0097819f0c6f5925e4e063d3fd474 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Sat, 1 Jul 2023 18:16:23 -0600 Subject: [PATCH 0043/1122] Move chain continuity checks into `scan_block_with_runner` In preparation for out-of-order range-based scanning, it is necessary to ensure that the size of the Sapling note commitment tree is carried along through the scan process and that stored blocks are always persisted with the updated note commitment tree size. --- zcash_client_backend/CHANGELOG.md | 25 +-- zcash_client_backend/src/data_api.rs | 121 +++++++++++-- zcash_client_backend/src/data_api/chain.rs | 144 ++++++---------- .../src/data_api/chain/error.rs | 156 ++--------------- zcash_client_backend/src/data_api/wallet.rs | 6 +- zcash_client_backend/src/lib.rs | 2 +- zcash_client_backend/src/welding_rig.rs | 162 ++++++++++++------ zcash_client_sqlite/CHANGELOG.md | 6 + zcash_client_sqlite/src/chain.rs | 16 +- zcash_client_sqlite/src/error.rs | 13 +- zcash_client_sqlite/src/lib.rs | 99 +++++------ zcash_client_sqlite/src/wallet.rs | 135 +++++++++++---- zcash_client_sqlite/src/wallet/init.rs | 49 +++++- zcash_client_sqlite/src/wallet/sapling.rs | 45 +++-- zcash_primitives/src/block.rs | 18 +- 15 files changed, 544 insertions(+), 453 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 8a6fc7f364..5df41eb405 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -11,14 +11,16 @@ and this library adheres to Rust's notion of - `impl Eq for zcash_client_backend::zip321::{Payment, TransactionRequest}` - `impl Debug` for `zcash_client_backend::{data_api::wallet::input_selection::Proposal, wallet::ReceivedSaplingNote}` - `zcash_client_backend::data_api`: - - `WalletRead::{fully_scanned_height, suggest_scan_ranges}` + - `WalletRead::{block_metadata, block_fully_scanned, suggest_scan_ranges}` - `WalletWrite::put_block` - `WalletCommitmentTrees` - `testing::MockWalletDb::new` - `NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` + - `BlockMetadata` + - `ScannedBlock` - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}`: - `zcash_client_backend::wallet::WalletSaplingOutput::note_commitment_tree_position` -- `zcash_client_backend::welding_rig::SyncError` +- `zcash_client_backend::welding_rig::ScanError` ### Changed - MSRV is now 1.65.0. @@ -56,27 +58,28 @@ and this library adheres to Rust's notion of - `zcash_client_backend::welding_rig::ScanningKey::sapling_nf` has been changed to take a note position instead of an incremental witness for the note. - Arguments to `zcash_client_backend::welding_rig::scan_block` have changed. This - method now takes an optional `CommitmentTreeMeta` argument instead of a base commitment - tree and incremental witnesses for each previously-known note. + method now takes an optional `BlockMetadata` argument instead of a base commitment + tree and incremental witnesses for each previously-known note. In addition, the + return type has now been updated to return a `Result` + in the case of scan failure. ### Removed - `zcash_client_backend::data_api`: - `WalletRead::get_all_nullifiers` - - `WalletRead::{get_commitment_tree, get_witnesses}` (use - `WalletRead::fully_scanned_height` instead). + - `WalletRead::{get_commitment_tree, get_witnesses}` have been removed + without replacement. The utility of these methods is now subsumed + by those available from the `WalletCommitmentTrees` trait. - `WalletWrite::advance_by_block` (use `WalletWrite::put_block` instead). - - The `commitment_tree` and `transactions` properties of the `PrunedBlock` - struct are now owned values instead of references, making this a fully owned type. - It is now parameterized by the nullifier type instead of by a lifetime for the - fields that were previously references. In addition, two new properties, - `sapling_commitment_tree_size` and `sapling_commitments` have been added. + - `PrunedBlock` has been replaced by `ScannedBlock` - `testing::MockWalletDb`, which is available under the `test-dependencies` feature flag, has been modified by the addition of a `sapling_tree` property. - `wallet::input_selection`: - `Proposal::target_height` (use `Proposal::min_target_height` instead). - `zcash_client_backend::data_api::chain::validate_chain` TODO: document how to handle validation given out-of-order blocks. +- `zcash_client_backend::data_api::chain::error::{ChainError, Cause}` have been + replaced by `zcash_client_backend::welding_rig::ScanError` - `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}` have been removed as individual incremental witnesses are no longer tracked on a per-note basis. The global note commitment tree for the wallet should be used diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 5b199b422f..6c5f8a115c 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -66,15 +66,17 @@ pub trait WalletRead { /// This will return `Ok(None)` if no block data is present in the database. fn block_height_extrema(&self) -> Result, Self::Error>; - /// Returns the height to which the wallet has been fully scanned. + /// Returns the available block metadata for the block at the specified height, if any. + fn block_metadata(&self, height: BlockHeight) -> Result, Self::Error>; + + /// Returns the metadata for the block at the height to which the wallet has been fully + /// scanned. /// /// This is the height for which the wallet has fully trial-decrypted this and all preceding /// blocks above the wallet's birthday height. Along with this height, this method returns /// metadata describing the state of the wallet's note commitment trees as of the end of that /// block. - fn fully_scanned_height( - &self, - ) -> Result, Self::Error>; + fn block_fully_scanned(&self) -> Result, Self::Error>; /// Returns a vector of suggested scan ranges based upon the current wallet state. /// @@ -248,17 +250,99 @@ pub trait WalletRead { ) -> Result, Self::Error>; } +/// Metadata describing the sizes of the zcash note commitment trees as of a particular block. +#[derive(Debug, Clone, Copy)] +pub struct BlockMetadata { + block_height: BlockHeight, + block_hash: BlockHash, + sapling_tree_size: u32, + //TODO: orchard_tree_size: u32 +} + +impl BlockMetadata { + /// Constructs a new [`BlockMetadata`] value from its constituent parts. + pub fn from_parts( + block_height: BlockHeight, + block_hash: BlockHash, + sapling_tree_size: u32, + ) -> Self { + Self { + block_height, + block_hash, + sapling_tree_size, + } + } + + /// Returns the block height. + pub fn block_height(&self) -> BlockHeight { + self.block_height + } + + /// Returns the hash of the block + pub fn block_hash(&self) -> BlockHash { + self.block_hash + } + + /// Returns the size of the Sapling note commitment tree as of the block that this + /// [`BlockMetadata`] describes. + pub fn sapling_tree_size(&self) -> u32 { + self.sapling_tree_size + } +} + /// The subset of information that is relevant to this wallet that has been /// decrypted and extracted from a [`CompactBlock`]. /// /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock -pub struct PrunedBlock { - pub block_height: BlockHeight, - pub block_hash: BlockHash, - pub block_time: u32, - pub transactions: Vec>, - pub sapling_commitment_tree_size: Option, - pub sapling_commitments: Vec<(sapling::Node, Retention)>, +pub struct ScannedBlock { + metadata: BlockMetadata, + block_time: u32, + transactions: Vec>, + sapling_commitments: Vec<(sapling::Node, Retention)>, +} + +impl ScannedBlock { + pub fn from_parts( + metadata: BlockMetadata, + block_time: u32, + transactions: Vec>, + sapling_commitments: Vec<(sapling::Node, Retention)>, + ) -> Self { + Self { + metadata, + block_time, + transactions, + sapling_commitments, + } + } + + pub fn height(&self) -> BlockHeight { + self.metadata.block_height + } + + pub fn block_hash(&self) -> BlockHash { + self.metadata.block_hash + } + + pub fn block_time(&self) -> u32 { + self.block_time + } + + pub fn metadata(&self) -> &BlockMetadata { + &self.metadata + } + + pub fn transactions(&self) -> &[WalletTx] { + &self.transactions + } + + pub fn sapling_commitments(&self) -> &[(sapling::Node, Retention)] { + &self.sapling_commitments + } + + pub fn take_sapling_commitments(self) -> Vec<(sapling::Node, Retention)> { + self.sapling_commitments + } } /// A transaction that was detected during scanning of the blockchain, @@ -404,7 +488,7 @@ pub trait WalletWrite: WalletRead { #[allow(clippy::type_complexity)] fn put_block( &mut self, - block: PrunedBlock, + block: ScannedBlock, ) -> Result, Self::Error>; /// Caches a decrypted transaction in the persistent wallet store. @@ -489,7 +573,7 @@ pub mod testing { }; use super::{ - chain, DecryptedTransaction, NullifierQuery, PrunedBlock, SentTransaction, + BlockMetadata, DecryptedTransaction, NullifierQuery, ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }; @@ -520,9 +604,14 @@ pub mod testing { Ok(None) } - fn fully_scanned_height( + fn block_metadata( &self, - ) -> Result, Self::Error> { + _height: BlockHeight, + ) -> Result, Self::Error> { + Ok(None) + } + + fn block_fully_scanned(&self) -> Result, Self::Error> { Ok(None) } @@ -667,7 +756,7 @@ pub mod testing { #[allow(clippy::type_complexity)] fn put_block( &mut self, - _block: PrunedBlock, + _block: ScannedBlock, ) -> Result, Self::Error> { Ok(vec![]) } diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index a554703077..8e1ceb8647 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -17,7 +17,6 @@ //! BlockSource, //! error::Error, //! scan_cached_blocks, -//! validate_chain, //! testing as chain_testing, //! }, //! testing, @@ -30,7 +29,7 @@ //! # test(); //! # } //! # -//! # fn test() -> Result<(), Error<(), Infallible, u32>> { +//! # fn test() -> Result<(), Error<(), Infallible>> { //! let network = Network::TestNetwork; //! let block_source = chain_testing::MockBlockSource; //! let mut db_data = testing::MockWalletDb::new(Network::TestNetwork); @@ -38,7 +37,7 @@ //! // 1) Download new CompactBlocks into block_source. //! // //! // 2) FIXME: Obtain necessary block metadata for continuity checking? -//! // +//! // //! // 3) Scan cached blocks. //! // //! // FIXME: update documentation on how to detect when a rewind is required. @@ -65,26 +64,7 @@ use crate::{ }; pub mod error; -use error::{ChainError, Error}; - -/// Metadata describing the sizes of the zcash note commitment trees as of a particular block. -pub struct CommitmentTreeMeta { - sapling_tree_size: u32, - //TODO: orchard_tree_size: u32 -} - -impl CommitmentTreeMeta { - /// Constructs a new [`CommitmentTreeMeta`] value from its constituent parts. - pub fn from_parts(sapling_tree_size: u32) -> Self { - Self { sapling_tree_size } - } - - /// Returns the size of the Sapling note commitment tree as of the block that this - /// [`CommitmentTreeMeta`] describes. - pub fn sapling_tree_size(&self) -> u32 { - self.sapling_tree_size - } -} +use error::Error; /// This trait provides sequential access to raw blockchain data via a callback-oriented /// API. @@ -99,14 +79,14 @@ pub trait BlockSource { /// as part of processing each row. /// * `NoteRefT`: the type of note identifiers in the wallet data store, for use in /// reporting errors related to specific notes. - fn with_blocks( + fn with_blocks( &self, from_height: Option, limit: Option, with_row: F, - ) -> Result<(), error::Error> + ) -> Result<(), error::Error> where - F: FnMut(CompactBlock) -> Result<(), error::Error>; + F: FnMut(CompactBlock) -> Result<(), error::Error>; } /// Scans at most `limit` new blocks added to the block source for any transactions received by the @@ -133,7 +113,7 @@ pub fn scan_cached_blocks( data_db: &mut DbT, from_height: Option, limit: Option, -) -> Result<(), Error> +) -> Result<(), Error> where ParamsT: consensus::Parameters + Send + 'static, BlockSourceT: BlockSource, @@ -169,74 +149,60 @@ where ); // Start at either the provided height, or where we synced up to previously. - let (scan_from, commitment_tree_meta) = from_height.map_or_else( - || { - data_db.fully_scanned_height().map_or_else( - |e| Err(Error::Wallet(e)), - |last_scanned| { - Ok(last_scanned.map_or_else(|| (None, None), |(h, m)| (Some(h + 1), Some(m)))) + let (scan_from, mut prior_block_metadata) = match from_height { + Some(h) => { + // if we are provided with a starting height, obtain the metadata for the previous + // block (if any is available) + ( + Some(h), + if h > BlockHeight::from(0) { + data_db.block_metadata(h - 1).map_err(Error::Wallet)? + } else { + None }, ) - }, - |h| Ok((Some(h), None)), - )?; + } + None => { + let last_scanned = data_db.block_fully_scanned().map_err(Error::Wallet)?; + last_scanned.map_or_else(|| (None, None), |m| (Some(m.block_height + 1), Some(m))) + } + }; - block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - scan_from, - limit, - |block: CompactBlock| { - add_block_to_runner(params, block, &mut batch_runner); - Ok(()) - }, - )?; + block_source.with_blocks::<_, DbT::Error>(scan_from, limit, |block: CompactBlock| { + add_block_to_runner(params, block, &mut batch_runner); + Ok(()) + })?; batch_runner.flush(); - let mut last_scanned = None; - block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - scan_from, - limit, - |block: CompactBlock| { - // block heights should be sequential within a single scan range - let block_height = block.height(); - if let Some(h) = last_scanned { - if block_height != h + 1 { - return Err(Error::Chain(ChainError::block_height_discontinuity( - block.height(), - h, - ))); - } - } - - let pruned_block = scan_block_with_runner( - params, - block, - &dfvks, - &sapling_nullifiers, - commitment_tree_meta.as_ref(), - Some(&mut batch_runner), - ) - .map_err(Error::Sync)?; + block_source.with_blocks::<_, DbT::Error>(scan_from, limit, |block: CompactBlock| { + let scanned_block = scan_block_with_runner( + params, + block, + &dfvks, + &sapling_nullifiers, + prior_block_metadata.as_ref(), + Some(&mut batch_runner), + ) + .map_err(Error::Scan)?; + + let spent_nf: Vec<&sapling::Nullifier> = scanned_block + .transactions + .iter() + .flat_map(|tx| tx.sapling_spends.iter().map(|spend| spend.nf())) + .collect(); - let spent_nf: Vec<&sapling::Nullifier> = pruned_block - .transactions + sapling_nullifiers.retain(|(_, nf)| !spent_nf.contains(&nf)); + sapling_nullifiers.extend(scanned_block.transactions.iter().flat_map(|tx| { + tx.sapling_outputs .iter() - .flat_map(|tx| tx.sapling_spends.iter().map(|spend| spend.nf())) - .collect(); - - sapling_nullifiers.retain(|(_, nf)| !spent_nf.contains(&nf)); - sapling_nullifiers.extend(pruned_block.transactions.iter().flat_map(|tx| { - tx.sapling_outputs - .iter() - .map(|out| (out.account(), *out.nf())) - })); + .map(|out| (out.account(), *out.nf())) + })); - data_db.put_block(pruned_block).map_err(Error::Wallet)?; - - last_scanned = Some(block_height); - Ok(()) - }, - )?; + prior_block_metadata = Some(*scanned_block.metadata()); + data_db.put_block(scanned_block).map_err(Error::Wallet)?; + Ok(()) + })?; Ok(()) } @@ -255,14 +221,14 @@ pub mod testing { impl BlockSource for MockBlockSource { type Error = Infallible; - fn with_blocks( + fn with_blocks( &self, _from_height: Option, _limit: Option, _with_row: F, - ) -> Result<(), Error> + ) -> Result<(), Error> where - F: FnMut(CompactBlock) -> Result<(), Error>, + F: FnMut(CompactBlock) -> Result<(), Error>, { Ok(()) } diff --git a/zcash_client_backend/src/data_api/chain/error.rs b/zcash_client_backend/src/data_api/chain/error.rs index b28380d13f..c1c78cf614 100644 --- a/zcash_client_backend/src/data_api/chain/error.rs +++ b/zcash_client_backend/src/data_api/chain/error.rs @@ -3,136 +3,11 @@ use std::error; use std::fmt::{self, Debug, Display}; -use zcash_primitives::{consensus::BlockHeight, sapling, transaction::TxId}; - -use crate::welding_rig::SyncError; - -/// The underlying cause of a [`ChainError`]. -#[derive(Copy, Clone, Debug)] -pub enum Cause { - /// The hash of the parent block given by a proposed new chain tip does not match the hash of - /// the current chain tip. - PrevHashMismatch, - - /// The block height field of the proposed new chain tip is not equal to the height of the - /// previous chain tip + 1. This variant stores a copy of the incorrect height value for - /// reporting purposes. - BlockHeightDiscontinuity(BlockHeight), - - /// The root of an output's witness tree in a newly arrived transaction does not correspond to - /// root of the stored commitment tree at the recorded height. - /// - /// This error is currently only produced when performing the slow checks that are enabled by - /// compiling with `-C debug-assertions`. - InvalidNewWitnessAnchor { - /// The id of the transaction containing the mismatched witness. - txid: TxId, - /// The index of the shielded output within the transaction where the witness root does not - /// match. - index: usize, - /// The root of the witness that failed to match the root of the current note commitment - /// tree. - node: sapling::Node, - }, - - /// The root of an output's witness tree in a previously stored transaction does not correspond - /// to root of the current commitment tree. - /// - /// This error is currently only produced when performing the slow checks that are enabled by - /// compiling with `-C debug-assertions`. - InvalidWitnessAnchor(NoteRef), -} - -/// Errors that may occur in chain scanning or validation. -#[derive(Copy, Clone, Debug)] -pub struct ChainError { - at_height: BlockHeight, - cause: Cause, -} - -impl fmt::Display for ChainError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match &self.cause { - Cause::PrevHashMismatch => write!( - f, - "The parent hash of proposed block does not correspond to the block hash at height {}.", - self.at_height - ), - Cause::BlockHeightDiscontinuity(h) => { - write!(f, "Block height discontinuity at height {}; next height is : {}", self.at_height, h) - } - Cause::InvalidNewWitnessAnchor { txid, index, node } => write!( - f, - "New witness for output {} in tx {} at height {} has incorrect anchor: {:?}", - index, txid, self.at_height, node, - ), - Cause::InvalidWitnessAnchor(id_note) => { - write!(f, "Witness for note {} has incorrect anchor for height {}", id_note, self.at_height) - } - } - } -} - -impl ChainError { - /// Constructs an error that indicates block hashes failed to chain. - /// - /// * `at_height` the height of the block whose parent hash does not match the hash of the - /// previous block - pub fn prev_hash_mismatch(at_height: BlockHeight) -> Self { - ChainError { - at_height, - cause: Cause::PrevHashMismatch, - } - } - - /// Constructs an error that indicates a gap in block heights. - /// - /// * `at_height` the height of the block being added to the chain. - /// * `prev_chain_tip` the height of the previous chain tip. - pub fn block_height_discontinuity(at_height: BlockHeight, prev_chain_tip: BlockHeight) -> Self { - ChainError { - at_height, - cause: Cause::BlockHeightDiscontinuity(prev_chain_tip), - } - } - - /// Constructs an error that indicates a mismatch between an updated note's witness and the - /// root of the current note commitment tree. - pub fn invalid_witness_anchor(at_height: BlockHeight, note_ref: NoteRef) -> Self { - ChainError { - at_height, - cause: Cause::InvalidWitnessAnchor(note_ref), - } - } - - /// Constructs an error that indicates a mismatch between a new note's witness and the root of - /// the current note commitment tree. - pub fn invalid_new_witness_anchor( - at_height: BlockHeight, - txid: TxId, - index: usize, - node: sapling::Node, - ) -> Self { - ChainError { - at_height, - cause: Cause::InvalidNewWitnessAnchor { txid, index, node }, - } - } - - /// Returns the block height at which this error was discovered. - pub fn at_height(&self) -> BlockHeight { - self.at_height - } - - /// Returns the cause of this error. - pub fn cause(&self) -> &Cause { - &self.cause - } -} +use crate::welding_rig::ScanError; /// Errors related to chain validation and scanning. #[derive(Debug)] -pub enum Error { +pub enum Error { /// An error that was produced by wallet operations in the course of scanning the chain. Wallet(WalletError), @@ -143,13 +18,10 @@ pub enum Error { /// A block that was received violated rules related to chain continuity or contained note /// commitments that could not be reconciled with the note commitment tree(s) maintained by the /// wallet. - Chain(ChainError), - - /// An error occorred in block scanning. - Sync(SyncError), + Scan(ScanError), } -impl fmt::Display for Error { +impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match &self { Error::Wallet(e) => { @@ -166,25 +38,17 @@ impl fmt::Display for Error { - write!(f, "{}", err) - } - Error::Sync(SyncError::SaplingTreeSizeUnknown(h)) => { - write!( - f, - "Sync failed due to missing Sapling note commitment tree size at height {}", - h - ) + Error::Scan(e) => { + write!(f, "Scanning produced the following error: {}", e) } } } } -impl error::Error for Error +impl error::Error for Error where WE: Debug + Display + error::Error + 'static, BE: Debug + Display + error::Error + 'static, - N: Debug + Display, { fn source(&self) -> Option<&(dyn error::Error + 'static)> { match &self { @@ -195,8 +59,8 @@ where } } -impl From> for Error { - fn from(e: ChainError) -> Self { - Error::Chain(e) +impl From for Error { + fn from(e: ScanError) -> Self { + Error::Scan(e) } } diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index c423c02eac..7f3d4ce6ac 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1,5 +1,5 @@ -use std::{convert::Infallible, num::NonZeroU32}; use std::fmt::Debug; +use std::{convert::Infallible, num::NonZeroU32}; use shardtree::{ShardStore, ShardTree, ShardTreeError}; use zcash_primitives::{ @@ -394,7 +394,7 @@ pub fn propose_shielding( input_selector: &InputsT, shielding_threshold: NonNegativeAmount, from_addrs: &[TransparentAddress], - min_confirmations: NonZeroU32 + min_confirmations: NonZeroU32, ) -> Result< Proposal, Error< @@ -497,7 +497,7 @@ where selected, usk.sapling(), &dfvk, - usize::try_from(u32::from(min_confirmations) - 1).unwrap() + usize::try_from(u32::from(min_confirmations) - 1).unwrap(), )? .ok_or(Error::NoteMismatch(selected.note_id))?; diff --git a/zcash_client_backend/src/lib.rs b/zcash_client_backend/src/lib.rs index f737116623..1cb87bc9f0 100644 --- a/zcash_client_backend/src/lib.rs +++ b/zcash_client_backend/src/lib.rs @@ -16,8 +16,8 @@ pub mod fees; pub mod keys; pub mod proto; pub mod scan; +pub mod scanning; pub mod wallet; -pub mod welding_rig; pub mod zip321; pub use decrypt::{decrypt_transaction, DecryptedOutput, TransferType}; diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index 79e3a4c394..a5d8e9c5d0 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -4,6 +4,7 @@ use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; +use std::fmt::{self, Debug}; use incrementalmerkletree::{Position, Retention}; use subtle::{ConditionallySelectable, ConstantTimeEq, CtOption}; @@ -20,8 +21,7 @@ use zcash_primitives::{ zip32::{sapling::DiversifiableFullViewingKey, AccountId, Scope}, }; -use crate::data_api::chain::CommitmentTreeMeta; -use crate::data_api::PrunedBlock; +use crate::data_api::{BlockMetadata, ScannedBlock}; use crate::{ proto::compact_formats::CompactBlock, scan::{Batch, BatchRunner, Tasks}, @@ -109,12 +109,42 @@ impl ScanningKey for SaplingIvk { fn sapling_nf(_key: &Self::SaplingNk, _note: &sapling::Note, _position: Position) {} } -/// Errors that can occur in block scanning. -#[derive(Debug)] -pub enum SyncError { +/// Errors that may occur in chain scanning +#[derive(Copy, Clone, Debug)] +pub enum ScanError { + /// The hash of the parent block given by a proposed new chain tip does not match the hash of + /// the current chain tip. + PrevHashMismatch { at_height: BlockHeight }, + + /// The block height field of the proposed new chain tip is not equal to the height of the + /// previous chain tip + 1. This variant stores a copy of the incorrect height value for + /// reporting purposes. + BlockHeightDiscontinuity { + previous_tip: BlockHeight, + new_height: BlockHeight, + }, + /// The size of the Sapling note commitment tree was not provided as part of a [`CompactBlock`] /// being scanned, making it impossible to construct the nullifier for a detected note. - SaplingTreeSizeUnknown(BlockHeight), + SaplingTreeSizeUnknown { at_height: BlockHeight }, +} + +impl fmt::Display for ScanError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self { + ScanError::PrevHashMismatch { at_height } => write!( + f, + "The parent hash of proposed block does not correspond to the block hash at height {}.", + at_height + ), + ScanError::BlockHeightDiscontinuity { previous_tip, new_height } => { + write!(f, "Block height discontinuity at height {}; next height is : {}", previous_tip, new_height) + } + ScanError::SaplingTreeSizeUnknown { at_height } => { + write!(f, "Unable to determine Sapling note commitment tree size at height {}", at_height) + } + } + } } /// Scans a [`CompactBlock`] with a set of [`ScanningKey`]s. @@ -145,14 +175,14 @@ pub fn scan_block( block: CompactBlock, vks: &[(&AccountId, &K)], sapling_nullifiers: &[(AccountId, sapling::Nullifier)], - initial_commitment_tree_meta: Option<&CommitmentTreeMeta>, -) -> Result, SyncError> { + prior_block_metadata: Option<&BlockMetadata>, +) -> Result, ScanError> { scan_block_with_runner::<_, _, ()>( params, block, vks, sapling_nullifiers, - initial_commitment_tree_meta, + prior_block_metadata, None, ) } @@ -204,13 +234,28 @@ pub(crate) fn scan_block_with_runner< block: CompactBlock, vks: &[(&AccountId, &K)], nullifiers: &[(AccountId, sapling::Nullifier)], - initial_commitment_tree_meta: Option<&CommitmentTreeMeta>, + prior_block_metadata: Option<&BlockMetadata>, mut batch_runner: Option<&mut TaggedBatchRunner>, -) -> Result, SyncError> { +) -> Result, ScanError> { let mut wtxs: Vec> = vec![]; let mut sapling_note_commitments: Vec<(sapling::Node, Retention)> = vec![]; - let block_height = block.height(); - let block_hash = block.hash(); + let cur_height = block.height(); + let cur_hash = block.hash(); + + if let Some(prev) = prior_block_metadata { + if cur_height != prev.block_height() + 1 { + return Err(ScanError::BlockHeightDiscontinuity { + previous_tip: prev.block_height(), + new_height: cur_height, + }); + } + + if block.prev_hash() != prev.block_hash() { + return Err(ScanError::PrevHashMismatch { + at_height: cur_height, + }); + } + } // It's possible to make progress without a Sapling tree position if we don't have any Sapling // notes in the block, since we only use the position for constructing nullifiers for our own @@ -235,7 +280,10 @@ pub(crate) fn scan_block_with_runner< Some(m.sapling_commitment_tree_size - block_note_count) } }) - .or_else(|| initial_commitment_tree_meta.map(|m| m.sapling_tree_size())); + .or_else(|| prior_block_metadata.map(|m| m.sapling_tree_size())) + .ok_or(ScanError::SaplingTreeSizeUnknown { + at_height: cur_height, + })?; let block_tx_count = block.vtx.len(); for (tx_idx, tx) in block.vtx.into_iter().enumerate() { @@ -283,7 +331,7 @@ pub(crate) fn scan_block_with_runner< .into_iter() .map(|output| { ( - SaplingDomain::for_height(params.clone(), block_height), + SaplingDomain::for_height(params.clone(), cur_height), CompactOutputDescription::try_from(output) .expect("Invalid output found in compact block decoding."), ) @@ -300,7 +348,7 @@ pub(crate) fn scan_block_with_runner< }) .collect::>(); - let mut decrypted = runner.collect_results(block_hash, txid); + let mut decrypted = runner.collect_results(cur_hash, txid); (0..decoded.len()) .map(|i| { decrypted.remove(&(txid, i)).map(|d_note| { @@ -347,7 +395,7 @@ pub(crate) fn scan_block_with_runner< let is_checkpoint = output_idx + 1 == decoded.len() && tx_idx + 1 == block_tx_count; let retention = match (dec_output.is_some(), is_checkpoint) { (is_marked, true) => Retention::Checkpoint { - id: block_height, + id: cur_height, is_marked, }, (true, false) => Retention::Marked, @@ -362,9 +410,9 @@ pub(crate) fn scan_block_with_runner< // - Notes created by consolidation transactions. // - Notes sent from one account to itself. let is_change = spent_from_accounts.contains(&account); - let note_commitment_tree_position = sapling_commitment_tree_size - .map(|s| Position::from(u64::from(s + u32::try_from(output_idx).unwrap()))) - .ok_or(SyncError::SaplingTreeSizeUnknown(block_height))?; + let note_commitment_tree_position = Position::from(u64::from( + sapling_commitment_tree_size + u32::try_from(output_idx).unwrap(), + )); let nf = K::sapling_nf(&nk, ¬e, note_commitment_tree_position); shielded_outputs.push(WalletSaplingOutput::from_parts( @@ -392,17 +440,15 @@ pub(crate) fn scan_block_with_runner< }); } - sapling_commitment_tree_size = sapling_commitment_tree_size.map(|s| s + tx_outputs_len); + sapling_commitment_tree_size += tx_outputs_len; } - Ok(PrunedBlock { - block_height, - block_hash, - block_time: block.time, - transactions: wtxs, - sapling_commitment_tree_size, - sapling_commitments: sapling_note_commitments, - }) + Ok(ScannedBlock::from_parts( + BlockMetadata::from_parts(cur_height, cur_hash, sapling_commitment_tree_size), + block.time, + wtxs, + sapling_note_commitments, + )) } #[cfg(test)] @@ -415,6 +461,7 @@ mod tests { use rand_core::{OsRng, RngCore}; use zcash_note_encryption::Domain; use zcash_primitives::{ + block::BlockHash, consensus::{BlockHeight, Network}, constants::SPENDING_KEY_GENERATOR, memo::MemoBytes, @@ -430,9 +477,9 @@ mod tests { }; use crate::{ - data_api::chain::CommitmentTreeMeta, + data_api::BlockMetadata, proto::compact_formats::{ - BlockMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, + self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }, scan::BatchRunner, }; @@ -479,6 +526,7 @@ mod tests { /// from a `lightwalletd` that is not currently tracking note commitment tree sizes. fn fake_compact_block( height: BlockHeight, + prev_hash: BlockHash, nf: Nullifier, dfvk: &DiversifiableFullViewingKey, value: Amount, @@ -510,6 +558,7 @@ mod tests { rng.fill_bytes(&mut hash); hash }, + prev_hash: prev_hash.0.to_vec(), height: height.into(), ..Default::default() }; @@ -543,7 +592,7 @@ mod tests { cb.vtx.push(tx); } - cb.block_metadata = initial_sapling_tree_size.map(|s| BlockMetadata { + cb.block_metadata = initial_sapling_tree_size.map(|s| compact::BlockMetadata { sapling_commitment_tree_size: s + cb .vtx .iter() @@ -564,6 +613,7 @@ mod tests { let cb = fake_compact_block( 1u32.into(), + BlockHash([0; 32]), Nullifier([0; 32]), &dfvk, Amount::from_u64(5).unwrap(), @@ -589,16 +639,20 @@ mod tests { None }; - let pruned_block = scan_block_with_runner( + let scanned_block = scan_block_with_runner( &Network::TestNetwork, cb, &[(&account, &dfvk)], &[], - Some(&CommitmentTreeMeta::from_parts(0)), + Some(&BlockMetadata::from_parts( + BlockHeight::from(0), + BlockHash([0u8; 32]), + 0, + )), batch_runner.as_mut(), ) .unwrap(); - let txs = pruned_block.transactions; + let txs = scanned_block.transactions(); assert_eq!(txs.len(), 1); let tx = &txs[0]; @@ -613,17 +667,17 @@ mod tests { Position::from(1) ); - assert_eq!(pruned_block.sapling_commitment_tree_size, Some(2)); + assert_eq!(scanned_block.metadata().sapling_tree_size(), 2); assert_eq!( - pruned_block - .sapling_commitments + scanned_block + .sapling_commitments() .iter() .map(|(_, retention)| *retention) .collect::>(), vec![ Retention::Ephemeral, Retention::Checkpoint { - id: pruned_block.block_height, + id: scanned_block.height(), is_marked: true } ] @@ -643,6 +697,7 @@ mod tests { let cb = fake_compact_block( 1u32.into(), + BlockHash([0; 32]), Nullifier([0; 32]), &dfvk, Amount::from_u64(5).unwrap(), @@ -668,7 +723,7 @@ mod tests { None }; - let pruned_block = scan_block_with_runner( + let scanned_block = scan_block_with_runner( &Network::TestNetwork, cb, &[(&AccountId::from(0), &dfvk)], @@ -677,7 +732,7 @@ mod tests { batch_runner.as_mut(), ) .unwrap(); - let txs = pruned_block.transactions; + let txs = scanned_block.transactions(); assert_eq!(txs.len(), 1); let tx = &txs[0]; @@ -689,8 +744,8 @@ mod tests { assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); assert_eq!( - pruned_block - .sapling_commitments + scanned_block + .sapling_commitments() .iter() .map(|(_, retention)| *retention) .collect::>(), @@ -698,7 +753,7 @@ mod tests { Retention::Ephemeral, Retention::Marked, Retention::Checkpoint { - id: pruned_block.block_height, + id: scanned_block.height(), is_marked: false } ] @@ -718,6 +773,7 @@ mod tests { let cb = fake_compact_block( 1u32.into(), + BlockHash([0; 32]), nf, &dfvk, Amount::from_u64(5).unwrap(), @@ -727,15 +783,9 @@ mod tests { assert_eq!(cb.vtx.len(), 2); let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; - let pruned_block = scan_block( - &Network::TestNetwork, - cb, - &vks[..], - &[(account, nf)], - Some(&CommitmentTreeMeta::from_parts(0)), - ) - .unwrap(); - let txs = pruned_block.transactions; + let scanned_block = + scan_block(&Network::TestNetwork, cb, &vks[..], &[(account, nf)], None).unwrap(); + let txs = scanned_block.transactions(); assert_eq!(txs.len(), 1); let tx = &txs[0]; @@ -747,15 +797,15 @@ mod tests { assert_eq!(tx.sapling_spends[0].account(), account); assert_eq!( - pruned_block - .sapling_commitments + scanned_block + .sapling_commitments() .iter() .map(|(_, retention)| *retention) .collect::>(), vec![ Retention::Ephemeral, Retention::Checkpoint { - id: pruned_block.block_height, + id: scanned_block.height(), is_marked: false } ] diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index b1e332913d..68ffd2ff31 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -19,10 +19,16 @@ and this library adheres to Rust's notion of a note could be spent with fewer than `min_confirmations` confirmations if the wallet did not contain enough observed blocks to satisfy the `min_confirmations` value specified; this situation is now treated as an error. +- A `BlockConflict` variant has been added to `zcash_client_sqlite::error::SqliteClientError` ### Removed - The empty `wallet::transact` module has been removed. +### Fixed +- Fixed an off-by-one error in the `BlockSource` implementation for the SQLite-backed + `BlockDb` block database which could result in blocks being skipped at the start of + scan ranges. + ## [0.7.1] - 2023-05-17 ### Fixed diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 4d968bb360..478a3bf45b 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -26,16 +26,16 @@ pub mod migrations; /// Starting at `from_height`, the `with_row` callback is invoked with each block retrieved from /// the backing store. If the `limit` value provided is `None`, all blocks are traversed up to the /// maximum height. -pub(crate) fn blockdb_with_blocks( +pub(crate) fn blockdb_with_blocks( block_source: &BlockDb, from_height: Option, limit: Option, mut with_row: F, -) -> Result<(), Error> +) -> Result<(), Error> where - F: FnMut(CompactBlock) -> Result<(), Error>, + F: FnMut(CompactBlock) -> Result<(), Error>, { - fn to_chain_error, N>(err: E) -> Error { + fn to_chain_error>(err: E) -> Error { Error::BlockSource(err.into()) } @@ -195,16 +195,16 @@ pub(crate) fn blockmetadb_find_block( /// the backing store. If the `limit` value provided is `None`, all blocks are traversed up to the /// maximum height for which metadata is available. #[cfg(feature = "unstable")] -pub(crate) fn fsblockdb_with_blocks( +pub(crate) fn fsblockdb_with_blocks( cache: &FsBlockDb, from_height: Option, limit: Option, mut with_block: F, -) -> Result<(), Error> +) -> Result<(), Error> where - F: FnMut(CompactBlock) -> Result<(), Error>, + F: FnMut(CompactBlock) -> Result<(), Error>, { - fn to_chain_error, N>(err: E) -> Error { + fn to_chain_error>(err: E) -> Error { Error::BlockSource(err.into()) } diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 6eb0939f24..db122f1a20 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -53,13 +53,15 @@ pub enum SqliteClientError { /// A received memo cannot be interpreted as a UTF-8 string. InvalidMemo(zcash_primitives::memo::Error), - /// A requested rewind would violate invariants of the - /// storage layer. The payload returned with this error is - /// (safe rewind height, requested height). + /// An attempt to update block data would overwrite the current hash for a block with a + /// different hash. This indicates that a required rewind was not performed. + BlockConflict(BlockHeight), + + /// A requested rewind would violate invariants of the storage layer. The payload returned with + /// this error is (safe rewind height, requested height). RequestedRewindInvalid(BlockHeight, BlockHeight), - /// The space of allocatable diversifier indices has been exhausted for - /// the given account. + /// The space of allocatable diversifier indices has been exhausted for the given account. DiversifierIndexOutOfRange, /// An error occurred deriving a spending key from a seed and an account @@ -115,6 +117,7 @@ impl fmt::Display for SqliteClientError { SqliteClientError::DbError(e) => write!(f, "{}", e), SqliteClientError::Io(e) => write!(f, "{}", e), SqliteClientError::InvalidMemo(e) => write!(f, "{}", e), + SqliteClientError::BlockConflict(h) => write!(f, "A block hash conflict occurred at height {}; rewind required.", u32::from(*h)), SqliteClientError::DiversifierIndexOutOfRange => write!(f, "The space of available diversifier indices is exhausted"), SqliteClientError::KeyDerivationError(acct_id) => write!(f, "Key derivation failed for account {:?}", acct_id), SqliteClientError::AccountIdDiscontinuity => write!(f, "Wallet account identifiers must be sequential."), diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 01c613c2da..c94cf91290 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -55,10 +55,9 @@ use zcash_primitives::{ use zcash_client_backend::{ address::{AddressMetadata, UnifiedAddress}, data_api::{ - self, - chain::{BlockSource, CommitmentTreeMeta}, - DecryptedTransaction, NullifierQuery, PoolType, PrunedBlock, Recipient, SentTransaction, - WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, + self, chain::BlockSource, BlockMetadata, DecryptedTransaction, NullifierQuery, PoolType, + Recipient, ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, WalletWrite, + SAPLING_SHARD_HEIGHT, }, keys::{UnifiedFullViewingKey, UnifiedSpendingKey}, proto::compact_formats::CompactBlock, @@ -85,7 +84,7 @@ pub mod wallet; /// this delta from the chain tip to be pruned. pub(crate) const PRUNING_HEIGHT: u32 = 100; -pub(crate) const SAPLING_TABLES_PREFIX: &'static str = "sapling"; +pub(crate) const SAPLING_TABLES_PREFIX: &str = "sapling"; /// A newtype wrapper for sqlite primary key values for the notes /// table. @@ -157,10 +156,12 @@ impl, P: consensus::Parameters> WalletRead for W wallet::block_height_extrema(self.conn.borrow()).map_err(SqliteClientError::from) } - fn fully_scanned_height( - &self, - ) -> Result, Self::Error> { - wallet::fully_scanned_height(self.conn.borrow()) + fn block_metadata(&self, height: BlockHeight) -> Result, Self::Error> { + wallet::block_metadata(self.conn.borrow(), height) + } + + fn block_fully_scanned(&self) -> Result, Self::Error> { + wallet::block_fully_scanned(self.conn.borrow()) } fn suggest_scan_ranges( @@ -183,6 +184,14 @@ impl, P: consensus::Parameters> WalletRead for W wallet::get_tx_height(self.conn.borrow(), txid).map_err(SqliteClientError::from) } + fn get_current_address( + &self, + account: AccountId, + ) -> Result, Self::Error> { + wallet::get_current_address(self.conn.borrow(), &self.params, account) + .map(|res| res.map(|(addr, _)| addr)) + } + fn get_unified_full_viewing_keys( &self, ) -> Result, Self::Error> { @@ -196,14 +205,6 @@ impl, P: consensus::Parameters> WalletRead for W wallet::get_account_for_ufvk(self.conn.borrow(), &self.params, ufvk) } - fn get_current_address( - &self, - account: AccountId, - ) -> Result, Self::Error> { - wallet::get_current_address(self.conn.borrow(), &self.params, account) - .map(|res| res.map(|(addr, _)| addr)) - } - fn is_valid_account_extfvk( &self, account: AccountId, @@ -220,10 +221,6 @@ impl, P: consensus::Parameters> WalletRead for W wallet::get_balance_at(self.conn.borrow(), account, anchor_height) } - fn get_transaction(&self, id_tx: i64) -> Result { - wallet::get_transaction(self.conn.borrow(), &self.params, id_tx) - } - fn get_memo(&self, id_note: Self::NoteRef) -> Result, Self::Error> { match id_note { NoteId::SentNoteId(id_note) => wallet::get_sent_memo(self.conn.borrow(), id_note), @@ -233,6 +230,10 @@ impl, P: consensus::Parameters> WalletRead for W } } + fn get_transaction(&self, id_tx: i64) -> Result { + wallet::get_transaction(self.conn.borrow(), &self.params, id_tx) + } + fn get_sapling_nullifiers( &self, query: NullifierQuery, @@ -390,25 +391,25 @@ impl WalletWrite for WalletDb ) } - #[tracing::instrument(skip_all, fields(height = u32::from(block.block_height)))] + #[tracing::instrument(skip_all, fields(height = u32::from(block.height())))] #[allow(clippy::type_complexity)] fn put_block( &mut self, - block: PrunedBlock, + block: ScannedBlock, ) -> Result, Self::Error> { self.transactionally(|wdb| { // Insert the block into the database. wallet::put_block( wdb.conn.0, - block.block_height, - block.block_hash, - block.block_time, - block.sapling_commitment_tree_size.map(|s| s.into()), + block.height(), + block.block_hash(), + block.block_time(), + block.metadata().sapling_tree_size(), )?; let mut wallet_note_ids = vec![]; - for tx in &block.transactions { - let tx_row = wallet::put_tx_meta(wdb.conn.0, tx, block.block_height)?; + for tx in block.transactions() { + let tx_row = wallet::put_tx_meta(wdb.conn.0, tx, block.height())?; // Mark notes as spent and remove them from the scanning cache for spend in &tx.sapling_spends { @@ -424,19 +425,19 @@ impl WalletWrite for WalletDb } } - let sapling_commitments_len = block.sapling_commitments.len(); - let mut sapling_commitments = block.sapling_commitments.into_iter(); + let block_height = block.height(); + let sapling_tree_size = block.metadata().sapling_tree_size(); + let sapling_commitments_len = block.sapling_commitments().len(); + let mut sapling_commitments = block.take_sapling_commitments().into_iter(); wdb.with_sapling_tree_mut::<_, _, SqliteClientError>(move |sapling_tree| { - if let Some(sapling_tree_size) = block.sapling_commitment_tree_size { - let start_position = Position::from(u64::from(sapling_tree_size)) - - u64::try_from(sapling_commitments_len).unwrap(); - sapling_tree.batch_insert(start_position, &mut sapling_commitments)?; - } + let start_position = Position::from(u64::from(sapling_tree_size)) + - u64::try_from(sapling_commitments_len).unwrap(); + sapling_tree.batch_insert(start_position, &mut sapling_commitments)?; Ok(()) })?; // Update now-expired transactions that didn't get mined. - wallet::update_expired_notes(wdb.conn.0, block.block_height)?; + wallet::update_expired_notes(wdb.conn.0, block_height)?; Ok(wallet_note_ids) }) @@ -688,17 +689,14 @@ impl BlockDb { impl BlockSource for BlockDb { type Error = SqliteClientError; - fn with_blocks( + fn with_blocks( &self, from_height: Option, limit: Option, with_row: F, - ) -> Result<(), data_api::chain::error::Error> + ) -> Result<(), data_api::chain::error::Error> where - F: FnMut( - CompactBlock, - ) - -> Result<(), data_api::chain::error::Error>, + F: FnMut(CompactBlock) -> Result<(), data_api::chain::error::Error>, { chain::blockdb_with_blocks(self, from_height, limit, with_row) } @@ -869,17 +867,14 @@ impl FsBlockDb { impl BlockSource for FsBlockDb { type Error = FsBlockDbError; - fn with_blocks( + fn with_blocks( &self, from_height: Option, limit: Option, with_row: F, - ) -> Result<(), data_api::chain::error::Error> + ) -> Result<(), data_api::chain::error::Error> where - F: FnMut( - CompactBlock, - ) - -> Result<(), data_api::chain::error::Error>, + F: FnMut(CompactBlock) -> Result<(), data_api::chain::error::Error>, { fsblockdb_with_blocks(self, from_height, limit, with_row) } @@ -967,7 +962,7 @@ mod tests { data_api::{WalletRead, WalletWrite}, keys::{sapling, UnifiedFullViewingKey}, proto::compact_formats::{ - BlockMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, + self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }, }; @@ -1112,7 +1107,7 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); - cb.block_metadata = Some(BlockMetadata { + cb.block_metadata = Some(compact::BlockMetadata { sapling_commitment_tree_size: initial_sapling_tree_size + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), ..Default::default() @@ -1203,7 +1198,7 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); - cb.block_metadata = Some(BlockMetadata { + cb.block_metadata = Some(compact::BlockMetadata { sapling_commitment_tree_size: initial_sapling_tree_size + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), ..Default::default() diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 722153dbf5..13d7ab8c01 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -65,9 +65,9 @@ //! - `memo` the shielded memo associated with the output, if any. use rusqlite::{self, named_params, OptionalExtension, ToSql}; -use std::collections::HashMap; use std::convert::TryFrom; use std::io::Cursor; +use std::{collections::HashMap, io}; use zcash_primitives::{ block::BlockHash, @@ -83,7 +83,7 @@ use zcash_primitives::{ use zcash_client_backend::{ address::{RecipientAddress, UnifiedAddress}, - data_api::{chain::CommitmentTreeMeta, PoolType, Recipient, SentTransactionOutput}, + data_api::{BlockMetadata, PoolType, Recipient, SentTransactionOutput}, encoding::AddressCodec, keys::UnifiedFullViewingKey, wallet::WalletTx, @@ -541,24 +541,88 @@ pub(crate) fn block_height_extrema( }) } -pub(crate) fn fully_scanned_height( +fn parse_block_metadata( + row: (BlockHeight, Vec, Option, Vec), +) -> Option> { + let (block_height, hash_data, sapling_tree_size_opt, sapling_tree) = row; + let sapling_tree_size = sapling_tree_size_opt.map(Ok).or_else(|| { + if sapling_tree == BLOCK_SAPLING_FRONTIER_ABSENT { + None + } else { + // parse the legacy commitment tree data + read_commitment_tree::< + zcash_primitives::sapling::Node, + _, + { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, + >(Cursor::new(sapling_tree)) + .map(|tree| Some(tree.size().try_into().unwrap())) + .map_err(SqliteClientError::from) + .transpose() + } + })?; + + let block_hash = BlockHash::try_from_slice(&hash_data).ok_or_else(|| { + SqliteClientError::from(io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid block hash length: {}", hash_data.len()), + )) + }); + + Some(sapling_tree_size.and_then(|sapling_tree_size| { + block_hash.map(|block_hash| { + BlockMetadata::from_parts(block_height, block_hash, sapling_tree_size) + }) + })) +} + +pub(crate) fn block_metadata( + conn: &rusqlite::Connection, + block_height: BlockHeight, +) -> Result, SqliteClientError> { + let res_opt = conn + .query_row( + "SELECT height, hash, sapling_commitment_tree_size, sapling_tree + FROM blocks + WHERE height = :block_height", + named_params![":block_height": u32::from(block_height)], + |row| { + let height: u32 = row.get(0)?; + let block_hash: Vec = row.get(1)?; + let sapling_tree_size: Option = row.get(2)?; + let sapling_tree: Vec = row.get(3)?; + Ok(( + BlockHeight::from(height), + block_hash, + sapling_tree_size, + sapling_tree, + )) + }, + ) + .optional()?; + + res_opt.and_then(parse_block_metadata).transpose() +} + +pub(crate) fn block_fully_scanned( conn: &rusqlite::Connection, -) -> Result, SqliteClientError> { +) -> Result, SqliteClientError> { // FIXME: this will need to be rewritten once out-of-order scan range suggestion // is implemented. let res_opt = conn .query_row( - "SELECT height, sapling_commitment_tree_size, sapling_tree + "SELECT height, hash, sapling_commitment_tree_size, sapling_tree FROM blocks ORDER BY height DESC LIMIT 1", [], |row| { - let max_height: u32 = row.get(0)?; - let sapling_tree_size: Option = row.get(1)?; - let sapling_tree: Vec = row.get(2)?; + let height: u32 = row.get(0)?; + let block_hash: Vec = row.get(1)?; + let sapling_tree_size: Option = row.get(2)?; + let sapling_tree: Vec = row.get(3)?; Ok(( - BlockHeight::from(max_height), + BlockHeight::from(height), + block_hash, sapling_tree_size, sapling_tree, )) @@ -566,32 +630,7 @@ pub(crate) fn fully_scanned_height( ) .optional()?; - res_opt - .and_then(|(max_height, sapling_tree_size, sapling_tree)| { - sapling_tree_size - .map(|s| Ok(CommitmentTreeMeta::from_parts(s))) - .or_else(|| { - if &sapling_tree == BLOCK_SAPLING_FRONTIER_ABSENT { - None - } else { - // parse the legacy commitment tree data - read_commitment_tree::< - zcash_primitives::sapling::Node, - _, - { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, - >(Cursor::new(sapling_tree)) - .map(|tree| { - Some(CommitmentTreeMeta::from_parts( - tree.size().try_into().unwrap(), - )) - }) - .map_err(SqliteClientError::from) - .transpose() - } - }) - .map(|meta_res| meta_res.map(|meta| (max_height, meta))) - }) - .transpose() + res_opt.and_then(parse_block_metadata).transpose() } /// Returns the block height at which the specified transaction was mined, @@ -834,12 +873,34 @@ pub(crate) fn get_transparent_balances( /// Inserts information about a scanned block into the database. pub(crate) fn put_block( - conn: &rusqlite::Connection, + conn: &rusqlite::Transaction<'_>, block_height: BlockHeight, block_hash: BlockHash, block_time: u32, - sapling_commitment_tree_size: Option, + sapling_commitment_tree_size: u32, ) -> Result<(), SqliteClientError> { + let block_hash_data = conn + .query_row( + "SELECT hash FROM blocks WHERE height = ?", + [u32::from(block_height)], + |row| row.get::<_, Vec>(0), + ) + .optional()?; + + // Ensure that in the case of an upsert, we don't overwrite block data + // with information for a block with a different hash. + if let Some(bytes) = block_hash_data { + let expected_hash = BlockHash::try_from_slice(&bytes).ok_or_else(|| { + SqliteClientError::CorruptedData(format!( + "Invalid block hash at height {}", + u32::from(block_height) + )) + })?; + if expected_hash != block_hash { + return Err(SqliteClientError::BlockConflict(block_height)); + } + } + let mut stmt_upsert_block = conn.prepare_cached( "INSERT INTO blocks ( height, diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index c730aea735..05ccb72a48 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -1,24 +1,29 @@ //! Functions for initializing the various databases. use either::Either; +use incrementalmerkletree::Retention; use std::{collections::HashMap, fmt, io}; use rusqlite::{self, types::ToSql}; use schemer::{Migrator, MigratorError}; use schemer_rusqlite::RusqliteAdapter; use secrecy::SecretVec; -use shardtree::ShardTreeError; +use shardtree::{ShardTree, ShardTreeError}; use uuid::Uuid; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight}, + merkle_tree::read_commitment_tree, + sapling, transaction::components::amount::BalanceError, zip32::AccountId, }; -use zcash_client_backend::keys::UnifiedFullViewingKey; +use zcash_client_backend::{data_api::SAPLING_SHARD_HEIGHT, keys::UnifiedFullViewingKey}; -use crate::{error::SqliteClientError, wallet, WalletDb}; +use crate::{error::SqliteClientError, wallet, WalletDb, SAPLING_TABLES_PREFIX}; + +use super::commitment_tree::SqliteShardStore; mod migrations; @@ -289,9 +294,21 @@ pub fn init_blocks_table( return Err(SqliteClientError::TableNotEmpty); } + let block_end_tree = + read_commitment_tree::( + sapling_tree, + ) + .map_err(|e| { + rusqlite::Error::FromSqlConversionFailure( + sapling_tree.len(), + rusqlite::types::Type::Blob, + Box::new(e), + ) + })?; + wdb.conn.0.execute( "INSERT INTO blocks (height, hash, time, sapling_tree) - VALUES (?, ?, ?, ?)", + VALUES (?, ?, ?, ?)", [ u32::from(height).to_sql()?, hash.0.to_sql()?, @@ -300,6 +317,26 @@ pub fn init_blocks_table( ], )?; + if let Some(nonempty_frontier) = block_end_tree.to_frontier().value() { + let shard_store = + SqliteShardStore::<_, sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( + wdb.conn.0, + SAPLING_TABLES_PREFIX, + )?; + let mut shard_tree: ShardTree< + _, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + > = ShardTree::new(shard_store, 100); + shard_tree.insert_frontier_nodes( + nonempty_frontier.clone(), + Retention::Checkpoint { + id: height, + is_marked: false, + }, + )?; + } + Ok(()) }) } @@ -1154,7 +1191,7 @@ mod tests { BlockHeight::from(1u32), BlockHash([1; 32]), 1, - &[], + &[0x0, 0x0, 0x0], ) .unwrap(); @@ -1164,7 +1201,7 @@ mod tests { BlockHeight::from(2u32), BlockHash([2; 32]), 2, - &[], + &[0x0, 0x0, 0x0], ) .unwrap_err(); } diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 85e101c738..a686ec6175 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -508,7 +508,7 @@ pub(crate) mod tests { BlockHeight::from(1u32), BlockHash([1; 32]), 1, - &[], + &[0x0, 0x0, 0x0], ) .unwrap(); @@ -562,7 +562,7 @@ pub(crate) mod tests { // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); - let (cb, _) = fake_compact_block( + let (mut cb, _) = fake_compact_block( sapling_activation_height(), BlockHash([0; 32]), &dfvk, @@ -588,14 +588,15 @@ pub(crate) mod tests { ); // Add more funds to the wallet in a second note - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + 1, cb.hash(), &dfvk, AddressType::DefaultExternal, value, 1, - ); + ) + .0; insert_into_cache(&db_cache, &cb); scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); @@ -639,14 +640,15 @@ pub(crate) mod tests { // Mine blocks SAPLING_ACTIVATION_HEIGHT + 2 to 9 until just before the second // note is verified for i in 2..10 { - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + i, cb.hash(), &dfvk, AddressType::DefaultExternal, value, i, - ); + ) + .0; insert_into_cache(&db_cache, &cb); } scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); @@ -673,14 +675,15 @@ pub(crate) mod tests { ); // Mine block 11 so that the second note becomes verified - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + 10, cb.hash(), &dfvk, AddressType::DefaultExternal, value, 11, - ); + ) + .0; insert_into_cache(&db_cache, &cb); scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); @@ -718,7 +721,7 @@ pub(crate) mod tests { // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); - let (cb, _) = fake_compact_block( + let (mut cb, _) = fake_compact_block( sapling_activation_height(), BlockHash([0; 32]), &dfvk, @@ -774,14 +777,15 @@ pub(crate) mod tests { // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 41 (that don't send us funds) // until just before the first transaction expires for i in 1..42 { - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + i, cb.hash(), &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, i, - ); + ) + .0; insert_into_cache(&db_cache, &cb); } scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); @@ -807,14 +811,15 @@ pub(crate) mod tests { ); // Mine block SAPLING_ACTIVATION_HEIGHT + 42 so that the first transaction expires - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + 42, cb.hash(), &ExtendedSpendingKey::master(&[42]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, 42, - ); + ) + .0; insert_into_cache(&db_cache, &cb); scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); @@ -851,7 +856,7 @@ pub(crate) mod tests { // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); - let (cb, _) = fake_compact_block( + let (mut cb, _) = fake_compact_block( sapling_activation_height(), BlockHash([0; 32]), &dfvk, @@ -922,14 +927,15 @@ pub(crate) mod tests { // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 42 (that don't send us funds) // so that the first transaction expires for i in 1..=42 { - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + i, cb.hash(), &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, i, - ); + ) + .0; insert_into_cache(&db_cache, &cb); } scan_cached_blocks(&network, &db_cache, &mut db_data, None, None).unwrap(); @@ -1073,7 +1079,7 @@ pub(crate) mod tests { let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet - let (cb, _) = fake_compact_block( + let (mut cb, _) = fake_compact_block( sapling_activation_height(), BlockHash([0; 32]), &dfvk, @@ -1085,14 +1091,15 @@ pub(crate) mod tests { // Add 10 dust notes to the wallet for i in 1..=10 { - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + i, cb.hash(), &dfvk, AddressType::DefaultExternal, Amount::from_u64(1000).unwrap(), i, - ); + ) + .0; insert_into_cache(&db_cache, &cb); } diff --git a/zcash_primitives/src/block.rs b/zcash_primitives/src/block.rs index 6271e05356..b9ef9a5ca0 100644 --- a/zcash_primitives/src/block.rs +++ b/zcash_primitives/src/block.rs @@ -39,10 +39,20 @@ impl BlockHash { /// /// This function will panic if the slice is not exactly 32 bytes. pub fn from_slice(bytes: &[u8]) -> Self { - assert_eq!(bytes.len(), 32); - let mut hash = [0; 32]; - hash.copy_from_slice(bytes); - BlockHash(hash) + Self::try_from_slice(bytes).unwrap() + } + + /// Constructs a [`BlockHash`] from the given slice. + /// + /// Returns `None` if `bytes` has any length other than 32 + pub fn try_from_slice(bytes: &[u8]) -> Option { + if bytes.len() == 32 { + let mut hash = [0; 32]; + hash.copy_from_slice(bytes); + Some(BlockHash(hash)) + } else { + None + } } } From 09a0096c748820b85d2997c0444b9ee6ac65b29a Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Sat, 1 Jul 2023 19:49:35 -0600 Subject: [PATCH 0044/1122] Use valid serialized CommitmentTree values for migration tests. --- zcash_client_sqlite/src/wallet/init.rs | 4 ++-- .../src/wallet/init/migrations/shardtree_support.rs | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 05ccb72a48..a5fc9f1d0a 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -916,7 +916,7 @@ mod tests { // add a sapling sent note wdb.conn.execute( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00')", + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'000000')", [], )?; @@ -1080,7 +1080,7 @@ mod tests { RecipientAddress::Transparent(*ufvk.default_address().0.transparent().unwrap()) .encode(&tests::network()); wdb.conn.execute( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00')", + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'000000')", [], )?; wdb.conn.execute( diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 08708d65cb..f9f13771f6 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -114,9 +114,6 @@ impl RusqliteMigration for Migration { while let Some(row) = block_rows.next()? { let block_height: u32 = row.get(0)?; let sapling_tree_data: Vec = row.get(1)?; - if sapling_tree_data == vec![0x00] { - continue; - } let block_end_tree = read_commitment_tree::< sapling::Node, From 42ed6ba2a1c02eadb86f6ede4fa692c1ba6ae175 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Sat, 1 Jul 2023 20:45:24 -0600 Subject: [PATCH 0045/1122] Rename `zcash_client_backend::welding_rig` to `zcash_client_backend::scanning` --- zcash_client_backend/CHANGELOG.md | 9 +++++---- zcash_client_backend/src/data_api/chain.rs | 2 +- zcash_client_backend/src/data_api/chain/error.rs | 2 +- zcash_client_backend/src/{welding_rig.rs => scanning.rs} | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) rename zcash_client_backend/src/{welding_rig.rs => scanning.rs} (99%) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 5df41eb405..06cf34db98 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -20,7 +20,7 @@ and this library adheres to Rust's notion of - `ScannedBlock` - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}`: - `zcash_client_backend::wallet::WalletSaplingOutput::note_commitment_tree_position` -- `zcash_client_backend::welding_rig::ScanError` +- `zcash_client_backend::scanning::ScanError` ### Changed - MSRV is now 1.65.0. @@ -55,9 +55,10 @@ and this library adheres to Rust's notion of - Arguments to `{propose_transaction, propose_shielding}` have changed. - `zcash_client_backend::wallet::ReceivedSaplingNote::note_commitment_tree_position` has replaced the `witness` field in the same struct. -- `zcash_client_backend::welding_rig::ScanningKey::sapling_nf` has been changed to +- `zcash_client_backend::welding_rig` has been renamed to `zcash_client_backend::scanning` +- `zcash_client_backend::scanning::ScanningKey::sapling_nf` has been changed to take a note position instead of an incremental witness for the note. -- Arguments to `zcash_client_backend::welding_rig::scan_block` have changed. This +- Arguments to `zcash_client_backend::scanning::scan_block` have changed. This method now takes an optional `BlockMetadata` argument instead of a base commitment tree and incremental witnesses for each previously-known note. In addition, the return type has now been updated to return a `Result` @@ -79,7 +80,7 @@ and this library adheres to Rust's notion of - `zcash_client_backend::data_api::chain::validate_chain` TODO: document how to handle validation given out-of-order blocks. - `zcash_client_backend::data_api::chain::error::{ChainError, Cause}` have been - replaced by `zcash_client_backend::welding_rig::ScanError` + replaced by `zcash_client_backend::scanning::ScanError` - `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}` have been removed as individual incremental witnesses are no longer tracked on a per-note basis. The global note commitment tree for the wallet should be used diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 8e1ceb8647..3546058e46 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -60,7 +60,7 @@ use crate::{ data_api::{NullifierQuery, WalletWrite}, proto::compact_formats::CompactBlock, scan::BatchRunner, - welding_rig::{add_block_to_runner, scan_block_with_runner}, + scanning::{add_block_to_runner, scan_block_with_runner}, }; pub mod error; diff --git a/zcash_client_backend/src/data_api/chain/error.rs b/zcash_client_backend/src/data_api/chain/error.rs index c1c78cf614..3a21884bc6 100644 --- a/zcash_client_backend/src/data_api/chain/error.rs +++ b/zcash_client_backend/src/data_api/chain/error.rs @@ -3,7 +3,7 @@ use std::error; use std::fmt::{self, Debug, Display}; -use crate::welding_rig::ScanError; +use crate::scanning::ScanError; /// Errors related to chain validation and scanning. #[derive(Debug)] diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/scanning.rs similarity index 99% rename from zcash_client_backend/src/welding_rig.rs rename to zcash_client_backend/src/scanning.rs index a5d8e9c5d0..0229f1a98d 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/scanning.rs @@ -40,7 +40,7 @@ use crate::{ /// nullifier for the note can also be obtained. /// /// [`CompactSaplingOutput`]: crate::proto::compact_formats::CompactSaplingOutput -/// [`scan_block`]: crate::welding_rig::scan_block +/// [`scan_block`]: crate::scanning::scan_block pub trait ScanningKey { /// The type representing the scope of the scanning key. type Scope: Clone + Eq + std::hash::Hash + Send + 'static; @@ -165,7 +165,7 @@ impl fmt::Display for ScanError { /// [`ExtendedFullViewingKey`]: zcash_primitives::zip32::ExtendedFullViewingKey /// [`SaplingIvk`]: zcash_primitives::sapling::SaplingIvk /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock -/// [`ScanningKey`]: crate::welding_rig::ScanningKey +/// [`ScanningKey`]: crate::scanning::ScanningKey /// [`CommitmentTree`]: zcash_primitives::sapling::CommitmentTree /// [`IncrementalWitness`]: zcash_primitives::sapling::IncrementalWitness /// [`WalletSaplingOutput`]: crate::wallet::WalletSaplingOutput From c363e71fa9d9595a00e25d26384afd16a12884d2 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 3 Jul 2023 16:19:13 -0600 Subject: [PATCH 0046/1122] Rename proto::compact::{BlockMetadata => ChainMetadata} --- zcash_client_backend/proto/compact_formats.proto | 4 ++-- zcash_client_backend/src/proto/compact_formats.rs | 6 +++--- zcash_client_backend/src/proto/service.rs | 2 +- zcash_client_backend/src/scanning.rs | 4 ++-- zcash_client_sqlite/src/lib.rs | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/zcash_client_backend/proto/compact_formats.proto b/zcash_client_backend/proto/compact_formats.proto index 740d7f7f3f..49a5c6f3f9 100644 --- a/zcash_client_backend/proto/compact_formats.proto +++ b/zcash_client_backend/proto/compact_formats.proto @@ -13,7 +13,7 @@ option swift_prefix = ""; // BlockMetadata represents information about a block that may not be // represented directly in the block data, but is instead derived from chain // data or other external sources. -message BlockMetadata { +message ChainMetadata { uint32 saplingCommitmentTreeSize = 1; // the size of the Sapling note commitment tree as of the end of this block uint32 orchardCommitmentTreeSize = 2; // the size of the Orchard note commitment tree as of the end of this block } @@ -30,7 +30,7 @@ message CompactBlock { uint32 time = 5; // Unix epoch time when the block was mined bytes header = 6; // (hash, prevHash, and time) OR (full header) repeated CompactTx vtx = 7; // zero or more compact transactions from this block - BlockMetadata blockMetadata = 8; // information about this block derived from the chain or other sources + ChainMetadata chainMetadata = 8; // information about the state of the chain as of this block } // CompactTx contains the minimum information for a wallet to know if this transaction diff --git a/zcash_client_backend/src/proto/compact_formats.rs b/zcash_client_backend/src/proto/compact_formats.rs index bf023eacfc..b03018059b 100644 --- a/zcash_client_backend/src/proto/compact_formats.rs +++ b/zcash_client_backend/src/proto/compact_formats.rs @@ -3,7 +3,7 @@ /// data or other external sources. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockMetadata { +pub struct ChainMetadata { /// the size of the Sapling note commitment tree as of the end of this block #[prost(uint32, tag = "1")] pub sapling_commitment_tree_size: u32, @@ -39,9 +39,9 @@ pub struct CompactBlock { /// zero or more compact transactions from this block #[prost(message, repeated, tag = "7")] pub vtx: ::prost::alloc::vec::Vec, - /// information about this block derived from the chain or other sources + /// information about the state of the chain as of this block #[prost(message, optional, tag = "8")] - pub block_metadata: ::core::option::Option, + pub chain_metadata: ::core::option::Option, } /// CompactTx contains the minimum information for a wallet to know if this transaction /// is relevant to it (either pays to it or spends from it) via shielded elements diff --git a/zcash_client_backend/src/proto/service.rs b/zcash_client_backend/src/proto/service.rs index 581762bb3b..17b375e3ed 100644 --- a/zcash_client_backend/src/proto/service.rs +++ b/zcash_client_backend/src/proto/service.rs @@ -3,7 +3,7 @@ /// data or other external sources. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockMetadata { +pub struct ChainMetadata { /// the size of the Sapling note commitment tree as of the end of this block #[prost(uint32, tag = "1")] pub sapling_commitment_tree_size: u32, diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index 0229f1a98d..194c3cc741 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -264,7 +264,7 @@ pub(crate) fn scan_block_with_runner< // the block, and we can't have a note of ours in a block with no outputs so treating the zero // default value from the protobuf as `None` is always correct. let mut sapling_commitment_tree_size = block - .block_metadata + .chain_metadata .as_ref() .and_then(|m| { if m.sapling_commitment_tree_size == 0 { @@ -592,7 +592,7 @@ mod tests { cb.vtx.push(tx); } - cb.block_metadata = initial_sapling_tree_size.map(|s| compact::BlockMetadata { + cb.chain_metadata = initial_sapling_tree_size.map(|s| compact::ChainMetadata { sapling_commitment_tree_size: s + cb .vtx .iter() diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index c94cf91290..00411d1a64 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -1107,7 +1107,7 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); - cb.block_metadata = Some(compact::BlockMetadata { + cb.chain_metadata = Some(compact::ChainMetadata { sapling_commitment_tree_size: initial_sapling_tree_size + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), ..Default::default() @@ -1198,7 +1198,7 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); - cb.block_metadata = Some(compact::BlockMetadata { + cb.chain_metadata = Some(compact::ChainMetadata { sapling_commitment_tree_size: initial_sapling_tree_size + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), ..Default::default() From c13c8c667896e8089ad49e88d5c033f9e844ffc4 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 3 Jul 2023 17:06:43 -0600 Subject: [PATCH 0047/1122] Address comments from code review. --- zcash_client_backend/CHANGELOG.md | 5 +- zcash_client_backend/build.rs | 4 + .../proto/compact_formats.proto | 4 +- zcash_client_backend/src/data_api.rs | 2 +- zcash_client_backend/src/data_api/chain.rs | 9 +- .../src/proto/compact_formats.rs | 4 +- zcash_client_backend/src/proto/service.rs | 13 --- zcash_client_backend/src/scanning.rs | 5 +- zcash_client_sqlite/src/error.rs | 4 +- zcash_client_sqlite/src/lib.rs | 8 +- zcash_client_sqlite/src/wallet.rs | 105 +++++++++--------- zcash_client_sqlite/src/wallet/init.rs | 4 +- .../init/migrations/shardtree_support.rs | 4 +- 13 files changed, 78 insertions(+), 93 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 06cf34db98..1cb28eaef9 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -46,7 +46,7 @@ and this library adheres to Rust's notion of respective `min_confirmations` arguments as `NonZeroU32` - `data_api::wallet::input_selection::InputSelector::{propose_transaction, propose_shielding}` now take their respective `min_confirmations` arguments as `NonZeroU32` - - A new `Sync` variant has been added to `data_api::chain::error::Error`. + - A new `Scan` variant has been added to `data_api::chain::error::Error`. - A new `SyncRequired` variant has been added to `data_api::wallet::input_selection::InputSelectorError`. - `zcash_client_backend::wallet`: - `SpendableNote` has been renamed to `ReceivedSaplingNote`. @@ -61,8 +61,7 @@ and this library adheres to Rust's notion of - Arguments to `zcash_client_backend::scanning::scan_block` have changed. This method now takes an optional `BlockMetadata` argument instead of a base commitment tree and incremental witnesses for each previously-known note. In addition, the - return type has now been updated to return a `Result` - in the case of scan failure. + return type has now been updated to return a `Result`. ### Removed diff --git a/zcash_client_backend/build.rs b/zcash_client_backend/build.rs index 271b0f781e..fdc201f572 100644 --- a/zcash_client_backend/build.rs +++ b/zcash_client_backend/build.rs @@ -45,6 +45,10 @@ fn build() -> io::Result<()> { // Build the gRPC types and client. tonic_build::configure() .build_server(false) + .extern_path( + ".cash.z.wallet.sdk.rpc.ChainMetadata", + "crate::proto::compact_formats::ChainMetadata", + ) .extern_path( ".cash.z.wallet.sdk.rpc.CompactBlock", "crate::proto::compact_formats::CompactBlock", diff --git a/zcash_client_backend/proto/compact_formats.proto b/zcash_client_backend/proto/compact_formats.proto index 49a5c6f3f9..1db1ecf695 100644 --- a/zcash_client_backend/proto/compact_formats.proto +++ b/zcash_client_backend/proto/compact_formats.proto @@ -10,9 +10,7 @@ option swift_prefix = ""; // Remember that proto3 fields are all optional. A field that is not present will be set to its zero value. // bytes fields of hashes are in canonical little-endian format. -// BlockMetadata represents information about a block that may not be -// represented directly in the block data, but is instead derived from chain -// data or other external sources. +// ChainMetadata represents information about the state of the chain as of a given block. message ChainMetadata { uint32 saplingCommitmentTreeSize = 1; // the size of the Sapling note commitment tree as of the end of this block uint32 orchardCommitmentTreeSize = 2; // the size of the Orchard note commitment tree as of the end of this block diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 6c5f8a115c..bd281e282b 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -340,7 +340,7 @@ impl ScannedBlock { &self.sapling_commitments } - pub fn take_sapling_commitments(self) -> Vec<(sapling::Node, Retention)> { + pub fn into_sapling_commitments(self) -> Vec<(sapling::Node, Retention)> { self.sapling_commitments } } diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 3546058e46..fab86eaf64 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -92,10 +92,11 @@ pub trait BlockSource { /// Scans at most `limit` new blocks added to the block source for any transactions received by the /// tracked accounts. /// -/// If the `from_height` argument is not `None`, then the block source will begin requesting blocks -/// from the provided block source at the specified height; if `from_height` is `None then this -/// will begin scanning at first block after the position to which the wallet has previously -/// fully scanned the chain, thereby beginning or continuing a linear scan over all blocks. +/// If the `from_height` argument is not `None`, then this method block source will begin +/// requesting blocks from the provided block source at the specified height; if `from_height` is +/// `None then this will begin scanning at first block after the position to which the wallet has +/// previously fully scanned the chain, thereby beginning or continuing a linear scan over all +/// blocks. /// /// This function will return without error after scanning at most `limit` new blocks, to enable /// the caller to update their UI with scanning progress. Repeatedly calling this function with diff --git a/zcash_client_backend/src/proto/compact_formats.rs b/zcash_client_backend/src/proto/compact_formats.rs index b03018059b..2e8a435db4 100644 --- a/zcash_client_backend/src/proto/compact_formats.rs +++ b/zcash_client_backend/src/proto/compact_formats.rs @@ -1,6 +1,4 @@ -/// BlockMetadata represents information about a block that may not be -/// represented directly in the block data, but is instead derived from chain -/// data or other external sources. +/// ChainMetadata represents information about the state of the chain as of a given block. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ChainMetadata { diff --git a/zcash_client_backend/src/proto/service.rs b/zcash_client_backend/src/proto/service.rs index 17b375e3ed..38b15abdbf 100644 --- a/zcash_client_backend/src/proto/service.rs +++ b/zcash_client_backend/src/proto/service.rs @@ -1,16 +1,3 @@ -/// BlockMetadata represents information about a block that may not be -/// represented directly in the block data, but is instead derived from chain -/// data or other external sources. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ChainMetadata { - /// the size of the Sapling note commitment tree as of the end of this block - #[prost(uint32, tag = "1")] - pub sapling_commitment_tree_size: u32, - /// the size of the Orchard note commitment tree as of the end of this block - #[prost(uint32, tag = "2")] - pub orchard_commitment_tree_size: u32, -} /// A BlockID message contains identifiers to select a block: a height or a /// hash. Specification by hash is not implemented, but may be in the future. #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index 194c3cc741..8eeb1b57e1 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -285,7 +285,7 @@ pub(crate) fn scan_block_with_runner< at_height: cur_height, })?; - let block_tx_count = block.vtx.len(); + let compact_block_tx_count = block.vtx.len(); for (tx_idx, tx) in block.vtx.into_iter().enumerate() { let txid = tx.txid(); @@ -392,7 +392,8 @@ pub(crate) fn scan_block_with_runner< { // Collect block note commitments let node = sapling::Node::from_cmu(&output.cmu); - let is_checkpoint = output_idx + 1 == decoded.len() && tx_idx + 1 == block_tx_count; + let is_checkpoint = + output_idx + 1 == decoded.len() && tx_idx + 1 == compact_block_tx_count; let retention = match (dec_output.is_some(), is_checkpoint) { (is_marked, true) => Retention::Checkpoint { id: cur_height, diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index db122f1a20..1dd14f1c23 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -9,7 +9,7 @@ use shardtree::ShardTreeError; use zcash_client_backend::encoding::{Bech32DecodeError, TransparentCodecError}; use zcash_primitives::{consensus::BlockHeight, zip32::AccountId}; -use crate::PRUNING_HEIGHT; +use crate::PRUNING_DEPTH; #[cfg(feature = "transparent-inputs")] use zcash_primitives::legacy::TransparentAddress; @@ -108,7 +108,7 @@ impl fmt::Display for SqliteClientError { SqliteClientError::InvalidNoteId => write!(f, "The note ID associated with an inserted witness must correspond to a received note."), SqliteClientError::RequestedRewindInvalid(h, r) => - write!(f, "A rewind must be either of less than {} blocks, or at least back to block {} for your wallet; the requested height was {}.", PRUNING_HEIGHT, h, r), + write!(f, "A rewind must be either of less than {} blocks, or at least back to block {} for your wallet; the requested height was {}.", PRUNING_DEPTH, h, r), SqliteClientError::Bech32DecodeError(e) => write!(f, "{}", e), #[cfg(feature = "transparent-inputs")] SqliteClientError::HdwalletError(e) => write!(f, "{:?}", e), diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 00411d1a64..b7351edf89 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -82,7 +82,7 @@ pub mod wallet; /// The maximum number of blocks the wallet is allowed to rewind. This is /// consistent with the bound in zcashd, and allows block data deeper than /// this delta from the chain tip to be pruned. -pub(crate) const PRUNING_HEIGHT: u32 = 100; +pub(crate) const PRUNING_DEPTH: u32 = 100; pub(crate) const SAPLING_TABLES_PREFIX: &str = "sapling"; @@ -428,7 +428,7 @@ impl WalletWrite for WalletDb let block_height = block.height(); let sapling_tree_size = block.metadata().sapling_tree_size(); let sapling_commitments_len = block.sapling_commitments().len(); - let mut sapling_commitments = block.take_sapling_commitments().into_iter(); + let mut sapling_commitments = block.into_sapling_commitments().into_iter(); wdb.with_sapling_tree_mut::<_, _, SqliteClientError>(move |sapling_tree| { let start_position = Position::from(u64::from(sapling_tree_size)) - u64::try_from(sapling_commitments_len).unwrap(); @@ -640,7 +640,7 @@ impl WalletCommitmentTrees for WalletDb WalletCommitmentTrees for WalletDb, Option, Vec), -) -> Option> { +) -> Result { let (block_height, hash_data, sapling_tree_size_opt, sapling_tree) = row; - let sapling_tree_size = sapling_tree_size_opt.map(Ok).or_else(|| { + let sapling_tree_size = sapling_tree_size_opt.map_or_else(|| { if sapling_tree == BLOCK_SAPLING_FRONTIER_ABSENT { - None + Err(SqliteClientError::CorruptedData("One of either the Sapling tree size or the legacy Sapling commitment tree must be present.".to_owned())) } else { // parse the legacy commitment tree data read_commitment_tree::< @@ -555,52 +555,50 @@ fn parse_block_metadata( _, { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, >(Cursor::new(sapling_tree)) - .map(|tree| Some(tree.size().try_into().unwrap())) + .map(|tree| tree.size().try_into().unwrap()) .map_err(SqliteClientError::from) - .transpose() } - })?; + }, Ok)?; let block_hash = BlockHash::try_from_slice(&hash_data).ok_or_else(|| { SqliteClientError::from(io::Error::new( io::ErrorKind::InvalidData, format!("Invalid block hash length: {}", hash_data.len()), )) - }); + })?; - Some(sapling_tree_size.and_then(|sapling_tree_size| { - block_hash.map(|block_hash| { - BlockMetadata::from_parts(block_height, block_hash, sapling_tree_size) - }) - })) + Ok(BlockMetadata::from_parts( + block_height, + block_hash, + sapling_tree_size, + )) } pub(crate) fn block_metadata( conn: &rusqlite::Connection, block_height: BlockHeight, ) -> Result, SqliteClientError> { - let res_opt = conn - .query_row( - "SELECT height, hash, sapling_commitment_tree_size, sapling_tree + conn.query_row( + "SELECT height, hash, sapling_commitment_tree_size, sapling_tree FROM blocks WHERE height = :block_height", - named_params![":block_height": u32::from(block_height)], - |row| { - let height: u32 = row.get(0)?; - let block_hash: Vec = row.get(1)?; - let sapling_tree_size: Option = row.get(2)?; - let sapling_tree: Vec = row.get(3)?; - Ok(( - BlockHeight::from(height), - block_hash, - sapling_tree_size, - sapling_tree, - )) - }, - ) - .optional()?; - - res_opt.and_then(parse_block_metadata).transpose() + named_params![":block_height": u32::from(block_height)], + |row| { + let height: u32 = row.get(0)?; + let block_hash: Vec = row.get(1)?; + let sapling_tree_size: Option = row.get(2)?; + let sapling_tree: Vec = row.get(3)?; + Ok(( + BlockHeight::from(height), + block_hash, + sapling_tree_size, + sapling_tree, + )) + }, + ) + .optional() + .map_err(SqliteClientError::from) + .and_then(|meta_row| meta_row.map(parse_block_metadata).transpose()) } pub(crate) fn block_fully_scanned( @@ -608,29 +606,28 @@ pub(crate) fn block_fully_scanned( ) -> Result, SqliteClientError> { // FIXME: this will need to be rewritten once out-of-order scan range suggestion // is implemented. - let res_opt = conn - .query_row( - "SELECT height, hash, sapling_commitment_tree_size, sapling_tree + conn.query_row( + "SELECT height, hash, sapling_commitment_tree_size, sapling_tree FROM blocks ORDER BY height DESC LIMIT 1", - [], - |row| { - let height: u32 = row.get(0)?; - let block_hash: Vec = row.get(1)?; - let sapling_tree_size: Option = row.get(2)?; - let sapling_tree: Vec = row.get(3)?; - Ok(( - BlockHeight::from(height), - block_hash, - sapling_tree_size, - sapling_tree, - )) - }, - ) - .optional()?; - - res_opt.and_then(parse_block_metadata).transpose() + [], + |row| { + let height: u32 = row.get(0)?; + let block_hash: Vec = row.get(1)?; + let sapling_tree_size: Option = row.get(2)?; + let sapling_tree: Vec = row.get(3)?; + Ok(( + BlockHeight::from(height), + block_hash, + sapling_tree_size, + sapling_tree, + )) + }, + ) + .optional() + .map_err(SqliteClientError::from) + .and_then(|meta_row| meta_row.map(parse_block_metadata).transpose()) } /// Returns the block height at which the specified transaction was mined, @@ -704,7 +701,7 @@ pub(crate) fn truncate_to_height( .map(|opt| opt.map_or_else(|| sapling_activation_height - 1, BlockHeight::from)) })?; - if block_height < last_scanned_height - PRUNING_HEIGHT { + if block_height < last_scanned_height - PRUNING_DEPTH { if let Some(h) = get_min_unspent_height(conn)? { if block_height > h { return Err(SqliteClientError::RequestedRewindInvalid(h, block_height)); diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index a5fc9f1d0a..66efda12e2 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -21,7 +21,7 @@ use zcash_primitives::{ use zcash_client_backend::{data_api::SAPLING_SHARD_HEIGHT, keys::UnifiedFullViewingKey}; -use crate::{error::SqliteClientError, wallet, WalletDb, SAPLING_TABLES_PREFIX}; +use crate::{error::SqliteClientError, wallet, WalletDb, PRUNING_DEPTH, SAPLING_TABLES_PREFIX}; use super::commitment_tree::SqliteShardStore; @@ -327,7 +327,7 @@ pub fn init_blocks_table( _, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, SAPLING_SHARD_HEIGHT, - > = ShardTree::new(shard_store, 100); + > = ShardTree::new(shard_store, PRUNING_DEPTH.try_into().unwrap()); shard_tree.insert_frontier_nodes( nonempty_frontier.clone(), Retention::Checkpoint { diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index f9f13771f6..8a238d00e4 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -23,7 +23,7 @@ use crate::{ commitment_tree::SqliteShardStore, init::{migrations::received_notes_nullable_nf, WalletMigrationError}, }, - SAPLING_TABLES_PREFIX, + PRUNING_DEPTH, SAPLING_TABLES_PREFIX, }; pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( @@ -103,7 +103,7 @@ impl RusqliteMigration for Migration { _, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, SAPLING_SHARD_HEIGHT, - > = ShardTree::new(shard_store, 100); + > = ShardTree::new(shard_store, PRUNING_DEPTH.try_into().unwrap()); // Insert all the tree information that we can get from block-end commitment trees { let mut stmt_blocks = transaction.prepare("SELECT height, sapling_tree FROM blocks")?; From 1e5b23aeba4e1da4f814fb75191c6f99cb724c3e Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 19 Jun 2023 19:05:35 -0600 Subject: [PATCH 0048/1122] zcash_client_backend: Add `put_sapling_subtree_roots` to `WalletCommitmentTrees` Also add the `zcash_client_sqlite` implementation & tests for the new method. --- zcash_client_backend/CHANGELOG.md | 1 + zcash_client_backend/src/data_api.rs | 32 ++- zcash_client_backend/src/data_api/chain.rs | 29 +++ zcash_client_sqlite/src/lib.rs | 42 +++- .../src/wallet/commitment_tree.rs | 184 ++++++++++++++++-- 5 files changed, 270 insertions(+), 18 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 1cb28eaef9..999d91860a 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -18,6 +18,7 @@ and this library adheres to Rust's notion of - `NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` - `BlockMetadata` - `ScannedBlock` + - `chain::CommitmentTreeRoot` - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}`: - `zcash_client_backend::wallet::WalletSaplingOutput::note_commitment_tree_position` - `zcash_client_backend::scanning::ScanError` diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index bd281e282b..7cc945f186 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -28,6 +28,8 @@ use crate::{ wallet::{ReceivedSaplingNote, WalletTransparentOutput, WalletTx}, }; +use self::chain::CommitmentTreeRoot; + pub mod chain; pub mod error; pub mod wallet; @@ -545,10 +547,18 @@ pub trait WalletCommitmentTrees { >, ) -> Result, E: From>; + + /// Adds a sequence of note commitment tree subtree roots to the data store. + fn put_sapling_subtree_roots( + &mut self, + start_index: u64, + roots: &[CommitmentTreeRoot], + ) -> Result<(), ShardTreeError>; } #[cfg(feature = "test-dependencies")] pub mod testing { + use incrementalmerkletree::Address; use secrecy::{ExposeSecret, SecretVec}; use shardtree::{MemoryShardStore, ShardTree, ShardTreeError}; use std::{collections::HashMap, convert::Infallible, ops::Range}; @@ -573,8 +583,9 @@ pub mod testing { }; use super::{ - BlockMetadata, DecryptedTransaction, NullifierQuery, ScannedBlock, SentTransaction, - WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, + chain::CommitmentTreeRoot, BlockMetadata, DecryptedTransaction, NullifierQuery, + ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, WalletWrite, + SAPLING_SHARD_HEIGHT, }; pub struct MockWalletDb { @@ -805,5 +816,22 @@ pub mod testing { { callback(&mut self.sapling_tree) } + + fn put_sapling_subtree_roots( + &mut self, + start_index: u64, + roots: &[CommitmentTreeRoot], + ) -> Result<(), ShardTreeError> { + self.with_sapling_tree_mut(|t| { + for (root, i) in roots.iter().zip(0u64..) { + let root_addr = + Address::from_parts(SAPLING_SHARD_HEIGHT.into(), start_index + i); + t.insert(root_addr, *root.root_hash())?; + } + Ok::<_, ShardTreeError>(()) + })?; + + Ok(()) + } } } diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index fab86eaf64..16d530ac7d 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -66,6 +66,35 @@ use crate::{ pub mod error; use error::Error; +/// A struct containing metadata about a subtree root of the note commitment tree. +/// +/// This stores the block height at which the leaf that completed the subtree was +/// added, and the root hash of the complete subtree. +pub struct CommitmentTreeRoot { + subtree_end_height: BlockHeight, + root_hash: H, +} + +impl CommitmentTreeRoot { + /// Construct a new `CommitmentTreeRoot` from its constituent parts. + pub fn from_parts(subtree_end_height: BlockHeight, root_hash: H) -> Self { + Self { + subtree_end_height, + root_hash, + } + } + + /// Returns the block height at which the leaf that completed the subtree was added. + pub fn subtree_end_height(&self) -> BlockHeight { + self.subtree_end_height + } + + /// Returns the root of the complete subtree. + pub fn root_hash(&self) -> &H { + &self.root_hash + } +} + /// This trait provides sequential access to raw blockchain data via a callback-oriented /// API. pub trait BlockSource { diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index b7351edf89..2f65053c45 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -36,6 +36,7 @@ use either::Either; use rusqlite::{self, Connection}; use secrecy::{ExposeSecret, SecretVec}; use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, io, ops::Range, path::Path}; +use wallet::commitment_tree::put_shard_roots; use incrementalmerkletree::Position; use shardtree::{ShardTree, ShardTreeError}; @@ -55,9 +56,10 @@ use zcash_primitives::{ use zcash_client_backend::{ address::{AddressMetadata, UnifiedAddress}, data_api::{ - self, chain::BlockSource, BlockMetadata, DecryptedTransaction, NullifierQuery, PoolType, - Recipient, ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, WalletWrite, - SAPLING_SHARD_HEIGHT, + self, + chain::{BlockSource, CommitmentTreeRoot}, + BlockMetadata, DecryptedTransaction, NullifierQuery, PoolType, Recipient, ScannedBlock, + SentTransaction, WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }, keys::{UnifiedFullViewingKey, UnifiedSpendingKey}, proto::compact_formats::CompactBlock, @@ -643,10 +645,31 @@ impl WalletCommitmentTrees for WalletDb], + ) -> Result<(), ShardTreeError> { + let tx = self + .conn + .transaction() + .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; + put_shard_roots::<_, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, SAPLING_SHARD_HEIGHT>( + &tx, + SAPLING_TABLES_PREFIX, + start_index, + roots, + )?; + tx.commit() + .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; + Ok(()) + } } impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb, P> { @@ -674,6 +697,19 @@ impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb], + ) -> Result<(), ShardTreeError> { + put_shard_roots::<_, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, SAPLING_SHARD_HEIGHT>( + self.conn.0, + SAPLING_TABLES_PREFIX, + start_index, + roots, + ) + } } /// A handle for the SQLite block source. diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 78548836aa..12cf243338 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -4,10 +4,15 @@ use std::{ collections::BTreeSet, io::{self, Cursor}, marker::PhantomData, + rc::Rc, }; +use zcash_client_backend::data_api::chain::CommitmentTreeRoot; -use incrementalmerkletree::{Address, Level, Position}; -use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore, TreeState}; +use incrementalmerkletree::{Address, Hashable, Level, Position, Retention}; +use shardtree::{ + Checkpoint, LocatedPrunableTree, LocatedTree, PrunableTree, RetentionFlags, ShardStore, + ShardTreeError, TreeState, +}; use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer}; @@ -257,23 +262,29 @@ type Error = Either; pub(crate) fn get_shard( conn: &rusqlite::Connection, table_prefix: &'static str, - shard_root: Address, + shard_root_addr: Address, ) -> Result>, Error> { conn.query_row( &format!( - "SELECT shard_data + "SELECT shard_data, root_hash FROM {}_tree_shards WHERE shard_index = :shard_index", table_prefix ), - named_params![":shard_index": shard_root.index()], - |row| row.get::<_, Vec>(0), + named_params![":shard_index": shard_root_addr.index()], + |row| Ok((row.get::<_, Vec>(0)?, row.get::<_, Option>>(1)?)), ) .optional() .map_err(Either::Right)? - .map(|shard_data| { + .map(|(shard_data, root_hash)| { let shard_tree = read_shard(&mut Cursor::new(shard_data)).map_err(Either::Left)?; - Ok(LocatedPrunableTree::from_parts(shard_root, shard_tree)) + let located_tree = LocatedPrunableTree::from_parts(shard_root_addr, shard_tree); + if let Some(root_hash_data) = root_hash { + let root_hash = H::read(Cursor::new(root_hash_data)).map_err(Either::Left)?; + Ok(located_tree.reannotate_root(Some(Rc::new(root_hash)))) + } else { + Ok(located_tree) + } }) .transpose() } @@ -746,18 +757,102 @@ pub(crate) fn truncate_checkpoints( Ok(()) } +pub(crate) fn put_shard_roots< + H: Hashable + HashSer + Clone + Eq, + const DEPTH: u8, + const SHARD_HEIGHT: u8, +>( + conn: &rusqlite::Transaction<'_>, + table_prefix: &'static str, + start_index: u64, + roots: &[CommitmentTreeRoot], +) -> Result<(), ShardTreeError> { + if roots.is_empty() { + // nothing to do + return Ok(()); + } + + // We treat the cap as a DEPTH-SHARD_HEIGHT tree so that we can make a batch insertion of + // root data using `Position::from(start_index)` as the starting position and treating the + // roots as level-0 leaves. + let cap = LocatedTree::from_parts( + Address::from_parts((DEPTH - SHARD_HEIGHT).into(), 0), + get_cap(conn, table_prefix).map_err(ShardTreeError::Storage)?, + ); + + let cap_result = cap + .batch_insert( + Position::from(start_index), + roots.iter().map(|r| { + ( + r.root_hash().clone(), + Retention::Checkpoint { + id: (), + is_marked: false, + }, + ) + }), + ) + .map_err(ShardTreeError::Insert)? + .expect("slice of inserted roots was verified to be nonempty"); + + put_cap(conn, table_prefix, cap_result.subtree.take_root()).map_err(ShardTreeError::Storage)?; + + for (root, i) in roots.iter().zip(0u64..) { + // We want to avoid deserializing the subtree just to annotate its root node, so we simply + // cache the downloaded root alongside of any already-persisted subtree. We will update the + // subtree data itself by reannotating the root node of the tree, handling conflicts, at + // the time that we deserialize the tree. + let mut stmt = conn + .prepare_cached(&format!( + "INSERT INTO {}_tree_shards (shard_index, subtree_end_height, root_hash, shard_data) + VALUES (:shard_index, :subtree_end_height, :root_hash, :shard_data) + ON CONFLICT (shard_index) DO UPDATE + SET subtree_end_height = :subtree_end_height, root_hash = :root_hash", + table_prefix + )) + .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; + + // The `shard_data` value will only be used in the case that no tree already exists. + let mut shard_data: Vec = vec![]; + let tree = PrunableTree::leaf((root.root_hash().clone(), RetentionFlags::EPHEMERAL)); + write_shard(&mut shard_data, &tree) + .map_err(|e| ShardTreeError::Storage(Either::Left(e)))?; + + let mut root_hash_data: Vec = vec![]; + root.root_hash() + .write(&mut root_hash_data) + .map_err(|e| ShardTreeError::Storage(Either::Left(e)))?; + + stmt.execute(named_params![ + ":shard_index": start_index + i, + ":subtree_end_height": u32::from(root.subtree_end_height()), + ":root_hash": root_hash_data, + ":shard_data": shard_data, + ]) + .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; + } + + Ok(()) +} + #[cfg(test)] mod tests { use tempfile::NamedTempFile; - use incrementalmerkletree::testing::{ - check_append, check_checkpoint_rewind, check_remove_mark, check_rewind_remove_mark, - check_root_hashes, check_witness_consistency, check_witnesses, + use incrementalmerkletree::{ + testing::{ + check_append, check_checkpoint_rewind, check_remove_mark, check_rewind_remove_mark, + check_root_hashes, check_witness_consistency, check_witnesses, + }, + Position, Retention, }; use shardtree::ShardTree; + use zcash_client_backend::data_api::chain::CommitmentTreeRoot; + use zcash_primitives::consensus::BlockHeight; use super::SqliteShardStore; - use crate::{tests, wallet::init::init_wallet_db, WalletDb}; + use crate::{tests, wallet::init::init_wallet_db, WalletDb, SAPLING_TABLES_PREFIX}; fn new_tree(m: usize) -> ShardTree, 4, 3> { let data_file = NamedTempFile::new().unwrap(); @@ -766,7 +861,8 @@ mod tests { init_wallet_db(&mut db_data, None).unwrap(); let store = - SqliteShardStore::<_, String, 3>::from_connection(db_data.conn, "sapling").unwrap(); + SqliteShardStore::<_, String, 3>::from_connection(db_data.conn, SAPLING_TABLES_PREFIX) + .unwrap(); ShardTree::new(store, m) } @@ -804,4 +900,66 @@ mod tests { fn rewind_remove_mark() { check_rewind_remove_mark(new_tree); } + + #[test] + fn put_shard_roots() { + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + data_file.keep().unwrap(); + + init_wallet_db(&mut db_data, None).unwrap(); + let tx = db_data.conn.transaction().unwrap(); + let store = + SqliteShardStore::<_, String, 3>::from_connection(&tx, SAPLING_TABLES_PREFIX).unwrap(); + + // introduce some roots + let roots = (0u32..4) + .into_iter() + .map(|idx| { + CommitmentTreeRoot::from_parts( + BlockHeight::from((idx + 1) * 3), + if idx == 3 { + "abcdefgh".to_string() + } else { + idx.to_string() + }, + ) + }) + .collect::>(); + super::put_shard_roots::<_, 6, 3>(store.conn, SAPLING_TABLES_PREFIX, 0, &roots).unwrap(); + + // simulate discovery of a note + let mut tree = ShardTree::<_, 6, 3>::new(store, 10); + tree.batch_insert( + Position::from(24), + ('a'..='h').into_iter().map(|c| { + ( + c.to_string(), + match c { + 'c' => Retention::Marked, + 'h' => Retention::Checkpoint { + id: BlockHeight::from(3), + is_marked: false, + }, + _ => Retention::Ephemeral, + }, + ) + }), + ) + .unwrap(); + + // construct a witness for the note + let witness = tree.witness(Position::from(26), 0).unwrap(); + assert_eq!( + witness.path_elems(), + &[ + "d", + "ab", + "efgh", + "2", + "01", + "________________________________" + ] + ); + } } From faccf56f04bf2c3ba67bd71c38cc93b4ab1363d4 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 26 Jun 2023 10:12:59 -0600 Subject: [PATCH 0049/1122] Split `PoolType` enum into `PoolType` and `ShieldedProtocol` There are cases where we wish to return informaiton that is relevant to a specific shielded protocol and `Transparent` is an invalid case. This is a minor preparatory refactoring that makes this distinction expressible. --- zcash_client_backend/CHANGELOG.md | 11 +++-- zcash_client_backend/src/data_api.rs | 13 ++++-- zcash_client_backend/src/data_api/wallet.rs | 40 +++++++++++-------- zcash_client_sqlite/src/lib.rs | 8 +++- zcash_client_sqlite/src/wallet.rs | 15 +++++-- .../wallet/init/migrations/ufvk_support.rs | 8 +++- .../init/migrations/v_transactions_net.rs | 5 ++- 7 files changed, 67 insertions(+), 33 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 999d91860a..c76af3ce26 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -11,14 +11,15 @@ and this library adheres to Rust's notion of - `impl Eq for zcash_client_backend::zip321::{Payment, TransactionRequest}` - `impl Debug` for `zcash_client_backend::{data_api::wallet::input_selection::Proposal, wallet::ReceivedSaplingNote}` - `zcash_client_backend::data_api`: + - `BlockMetadata` + - `NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` + - `ScannedBlock` + - `ShieldedProtocol` - `WalletRead::{block_metadata, block_fully_scanned, suggest_scan_ranges}` - `WalletWrite::put_block` - `WalletCommitmentTrees` - - `testing::MockWalletDb::new` - - `NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` - - `BlockMetadata` - - `ScannedBlock` - `chain::CommitmentTreeRoot` + - `testing::MockWalletDb::new` - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}`: - `zcash_client_backend::wallet::WalletSaplingOutput::note_commitment_tree_position` - `zcash_client_backend::scanning::ScanError` @@ -49,6 +50,8 @@ and this library adheres to Rust's notion of now take their respective `min_confirmations` arguments as `NonZeroU32` - A new `Scan` variant has been added to `data_api::chain::error::Error`. - A new `SyncRequired` variant has been added to `data_api::wallet::input_selection::InputSelectorError`. + - The variants of the `PoolType` enum have changed; the `PoolType::Sapling` variant has been + removed in favor of a `PoolType::Shielded` variant that wraps a `ShieldedProtocol` value. - `zcash_client_backend::wallet`: - `SpendableNote` has been renamed to `ReceivedSaplingNote`. - Arguments to `WalletSaplingOutput::from_parts` have changed. diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 7cc945f186..bef383e2b6 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -372,14 +372,21 @@ pub struct SentTransaction<'a> { pub utxos_spent: Vec, } +/// A shielded transfer protocol supported by the wallet. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum ShieldedProtocol { + /// The Sapling protocol + Sapling, + // TODO: Orchard +} + /// A value pool to which the wallet supports sending transaction outputs. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum PoolType { /// The transparent value pool Transparent, - /// The Sapling value pool - Sapling, - // TODO: Orchard + /// A shielded value pool. + Shielded(ShieldedProtocol), } /// A type that represents the recipient of a transaction output; a recipient address (and, for diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 7f3d4ce6ac..60ef900eea 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -37,6 +37,8 @@ use crate::{ pub mod input_selection; use input_selection::{GreedyInputSelector, GreedyInputSelectorError, InputSelector}; +use super::ShieldedProtocol; + #[cfg(feature = "transparent-inputs")] use { crate::wallet::WalletTransparentOutput, @@ -550,7 +552,7 @@ where payment.memo.clone().unwrap_or_else(MemoBytes::empty), )?; sapling_output_meta.push(( - Recipient::Unified(ua.clone(), PoolType::Sapling), + Recipient::Unified(ua.clone(), PoolType::Shielded(ShieldedProtocol::Sapling)), payment.amount, payment.memo.clone(), )); @@ -589,7 +591,10 @@ where MemoBytes::empty(), )?; sapling_output_meta.push(( - Recipient::InternalAccount(account, PoolType::Sapling), + Recipient::InternalAccount( + account, + PoolType::Shielded(ShieldedProtocol::Sapling), + ), *amount, change_memo.clone(), )) @@ -610,20 +615,23 @@ where .output_index(i) .expect("An output should exist in the transaction for each shielded payment."); - let received_as = - if let Recipient::InternalAccount(account, PoolType::Sapling) = recipient { - tx.sapling_bundle().and_then(|bundle| { - try_sapling_note_decryption( - params, - proposal.min_target_height(), - &internal_ivk, - &bundle.shielded_outputs()[output_index], - ) - .map(|(note, _, _)| (account, note)) - }) - } else { - None - }; + let received_as = if let Recipient::InternalAccount( + account, + PoolType::Shielded(ShieldedProtocol::Sapling), + ) = recipient + { + tx.sapling_bundle().and_then(|bundle| { + try_sapling_note_decryption( + params, + proposal.min_target_height(), + &internal_ivk, + &bundle.shielded_outputs()[output_index], + ) + .map(|(note, _, _)| (account, note)) + }) + } else { + None + }; SentTransactionOutput::from_parts(output_index, recipient, value, memo, received_as) }); diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 2f65053c45..c3d9b04bd5 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -59,7 +59,8 @@ use zcash_client_backend::{ self, chain::{BlockSource, CommitmentTreeRoot}, BlockMetadata, DecryptedTransaction, NullifierQuery, PoolType, Recipient, ScannedBlock, - SentTransaction, WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, + SentTransaction, ShieldedProtocol, WalletCommitmentTrees, WalletRead, WalletWrite, + SAPLING_SHARD_HEIGHT, }, keys::{UnifiedFullViewingKey, UnifiedSpendingKey}, proto::compact_formats::CompactBlock, @@ -459,7 +460,10 @@ impl WalletWrite for WalletDb let recipient = if output.transfer_type == TransferType::Outgoing { Recipient::Sapling(output.note.recipient()) } else { - Recipient::InternalAccount(output.account, PoolType::Sapling) + Recipient::InternalAccount( + output.account, + PoolType::Shielded(ShieldedProtocol::Sapling) + ) }; wallet::put_sent_output( diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 6b103cfc39..0e14588941 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -66,8 +66,11 @@ use rusqlite::{self, named_params, OptionalExtension, ToSql}; use std::convert::TryFrom; -use std::io::Cursor; -use std::{collections::HashMap, io}; +use std::{ + collections::HashMap, + io::{self, Cursor}, +}; +use zcash_client_backend::data_api::ShieldedProtocol; use zcash_primitives::{ block::BlockHash, @@ -116,7 +119,7 @@ pub(crate) fn pool_code(pool_type: PoolType) -> i64 { // implementation detail. match pool_type { PoolType::Transparent => 0i64, - PoolType::Sapling => 2i64, + PoolType::Shielded(ShieldedProtocol::Sapling) => 2i64, } } @@ -1119,7 +1122,11 @@ fn recipient_params( ) -> (Option, Option, PoolType) { match to { Recipient::Transparent(addr) => (Some(addr.encode(params)), None, PoolType::Transparent), - Recipient::Sapling(addr) => (Some(addr.encode(params)), None, PoolType::Sapling), + Recipient::Sapling(addr) => ( + Some(addr.encode(params)), + None, + PoolType::Shielded(ShieldedProtocol::Sapling), + ), Recipient::Unified(addr, pool) => (Some(addr.encode(params)), None, *pool), Recipient::InternalAccount(id, pool) => (None, Some(u32::from(*id)), *pool), } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/ufvk_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/ufvk_support.rs index def63a91d4..656281b704 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/ufvk_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/ufvk_support.rs @@ -8,7 +8,9 @@ use secrecy::{ExposeSecret, SecretVec}; use uuid::Uuid; use zcash_client_backend::{ - address::RecipientAddress, data_api::PoolType, keys::UnifiedSpendingKey, + address::RecipientAddress, + data_api::{PoolType, ShieldedProtocol}, + keys::UnifiedSpendingKey, }; use zcash_primitives::{consensus, zip32::AccountId}; @@ -229,7 +231,9 @@ impl RusqliteMigration for Migration

{ )) })?; let output_pool = match decoded_address { - RecipientAddress::Shielded(_) => Ok(pool_code(PoolType::Sapling)), + RecipientAddress::Shielded(_) => { + Ok(pool_code(PoolType::Shielded(ShieldedProtocol::Sapling))) + } RecipientAddress::Transparent(_) => Ok(pool_code(PoolType::Transparent)), RecipientAddress::Unified(_) => Err(WalletMigrationError::CorruptedData( "Unified addresses should not yet appear in the sent_notes table." diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs index fc3ab73786..7f82cea4a4 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs @@ -6,9 +6,10 @@ use rusqlite::{self, named_params}; use schemer; use schemer_rusqlite::RusqliteMigration; use uuid::Uuid; +use zcash_client_backend::data_api::{PoolType, ShieldedProtocol}; use super::add_transaction_views; -use crate::wallet::{init::WalletMigrationError, pool_code, PoolType}; +use crate::wallet::{init::WalletMigrationError, pool_code}; pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( 0x2aa4d24f, @@ -48,7 +49,7 @@ impl RusqliteMigration for Migration { SELECT tx, :output_pool, output_index, from_account, from_account, value FROM sent_notes", named_params![ - ":output_pool": &pool_code(PoolType::Sapling) + ":output_pool": &pool_code(PoolType::Shielded(ShieldedProtocol::Sapling)) ] )?; From 98147e0d7dda54ce451a28e8f8cf87d2571e19ba Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 26 Jun 2023 10:12:59 -0600 Subject: [PATCH 0050/1122] Make `from_height` and `limit` parameters to `scan_cached_blocks` non-optional. --- zcash_client_backend/CHANGELOG.md | 3 +- zcash_client_backend/src/data_api.rs | 1 - zcash_client_backend/src/data_api/chain.rs | 124 ++++++++++++--------- zcash_client_backend/src/scanning.rs | 105 ++++++++++++----- zcash_client_sqlite/src/chain.rs | 123 +++++++++++++++++--- zcash_client_sqlite/src/wallet.rs | 6 +- zcash_client_sqlite/src/wallet/sapling.rs | 119 +++++++++++++++++--- 7 files changed, 363 insertions(+), 118 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index c76af3ce26..618bf2c887 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -36,7 +36,8 @@ and this library adheres to Rust's notion of and its signature has changed; it now subsumes the removed `WalletRead::get_all_nullifiers`. - `WalletRead::get_target_and_anchor_heights` now takes its argument as a `NonZeroU32` - `chain::scan_cached_blocks` now takes a `from_height` argument that - permits the caller to control the starting position of the scan range. + permits the caller to control the starting position of the scan range + In addition, the `limit` parameter is now required. - A new `CommitmentTree` variant has been added to `data_api::error::Error` - `data_api::wallet::{create_spend_to_address, create_proposed_transaction, shield_transparent_funds}` all now require that `WalletCommitmentTrees` be diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index bef383e2b6..216fc7174b 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -494,7 +494,6 @@ pub trait WalletWrite: WalletRead { /// Updates the state of the wallet database by persisting the provided block information, /// along with the note commitments that were detected when scanning the block for transactions /// pertaining to this wallet. - #[allow(clippy::type_complexity)] fn put_block( &mut self, block: ScannedBlock, diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 16d530ac7d..29e72b1a58 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -45,7 +45,7 @@ //! // At this point, the cache and scanned data are locally consistent (though not //! // necessarily consistent with the latest chain tip - this would be discovered the //! // next time this codepath is executed after new blocks are received). -//! scan_cached_blocks(&network, &block_source, &mut db_data, None, None) +//! scan_cached_blocks(&network, &block_source, &mut db_data, BlockHeight::from(0), 10) //! # } //! # } //! ``` @@ -60,9 +60,11 @@ use crate::{ data_api::{NullifierQuery, WalletWrite}, proto::compact_formats::CompactBlock, scan::BatchRunner, - scanning::{add_block_to_runner, scan_block_with_runner}, + scanning::{add_block_to_runner, check_continuity, scan_block_with_runner}, }; +use super::BlockMetadata; + pub mod error; use error::Error; @@ -141,8 +143,8 @@ pub fn scan_cached_blocks( params: &ParamsT, block_source: &BlockSourceT, data_db: &mut DbT, - from_height: Option, - limit: Option, + from_height: BlockHeight, + limit: u32, ) -> Result<(), Error> where ParamsT: consensus::Parameters + Send + 'static, @@ -178,61 +180,81 @@ where .map(|(tag, ivk)| (tag, PreparedIncomingViewingKey::new(&ivk))), ); - // Start at either the provided height, or where we synced up to previously. - let (scan_from, mut prior_block_metadata) = match from_height { - Some(h) => { - // if we are provided with a starting height, obtain the metadata for the previous - // block (if any is available) - ( - Some(h), - if h > BlockHeight::from(0) { - data_db.block_metadata(h - 1).map_err(Error::Wallet)? - } else { - None - }, - ) - } - None => { - let last_scanned = data_db.block_fully_scanned().map_err(Error::Wallet)?; - last_scanned.map_or_else(|| (None, None), |m| (Some(m.block_height + 1), Some(m))) - } + let mut prior_block_metadata = if from_height > BlockHeight::from(0) { + data_db + .block_metadata(from_height - 1) + .map_err(Error::Wallet)? + } else { + None }; - block_source.with_blocks::<_, DbT::Error>(scan_from, limit, |block: CompactBlock| { - add_block_to_runner(params, block, &mut batch_runner); - Ok(()) - })?; + let mut continuity_check_metadata = prior_block_metadata; + block_source.with_blocks::<_, DbT::Error>( + Some(from_height), + Some(limit), + |block: CompactBlock| { + // check block continuity + if let Some(scan_error) = check_continuity(&block, continuity_check_metadata.as_ref()) { + return Err(Error::Scan(scan_error)); + } + continuity_check_metadata = continuity_check_metadata.as_ref().map(|m| { + BlockMetadata::from_parts( + block.height(), + block.hash(), + block + .chain_metadata + .as_ref() + .map(|m| m.sapling_commitment_tree_size) + .unwrap_or_else(|| { + m.sapling_tree_size() + + u32::try_from( + block.vtx.iter().map(|tx| tx.outputs.len()).sum::(), + ) + .unwrap() + }), + ) + }); - batch_runner.flush(); + add_block_to_runner(params, block, &mut batch_runner); + + Ok(()) + }, + )?; - block_source.with_blocks::<_, DbT::Error>(scan_from, limit, |block: CompactBlock| { - let scanned_block = scan_block_with_runner( - params, - block, - &dfvks, - &sapling_nullifiers, - prior_block_metadata.as_ref(), - Some(&mut batch_runner), - ) - .map_err(Error::Scan)?; + batch_runner.flush(); - let spent_nf: Vec<&sapling::Nullifier> = scanned_block - .transactions - .iter() - .flat_map(|tx| tx.sapling_spends.iter().map(|spend| spend.nf())) - .collect(); + block_source.with_blocks::<_, DbT::Error>( + Some(from_height), + Some(limit), + |block: CompactBlock| { + let scanned_block = scan_block_with_runner( + params, + block, + &dfvks, + &sapling_nullifiers, + prior_block_metadata.as_ref(), + Some(&mut batch_runner), + ) + .map_err(Error::Scan)?; - sapling_nullifiers.retain(|(_, nf)| !spent_nf.contains(&nf)); - sapling_nullifiers.extend(scanned_block.transactions.iter().flat_map(|tx| { - tx.sapling_outputs + let spent_nf: Vec<&sapling::Nullifier> = scanned_block + .transactions .iter() - .map(|out| (out.account(), *out.nf())) - })); + .flat_map(|tx| tx.sapling_spends.iter().map(|spend| spend.nf())) + .collect(); + + sapling_nullifiers.retain(|(_, nf)| !spent_nf.contains(&nf)); + sapling_nullifiers.extend(scanned_block.transactions.iter().flat_map(|tx| { + tx.sapling_outputs + .iter() + .map(|out| (out.account(), *out.nf())) + })); - prior_block_metadata = Some(*scanned_block.metadata()); - data_db.put_block(scanned_block).map_err(Error::Wallet)?; - Ok(()) - })?; + prior_block_metadata = Some(*scanned_block.metadata()); + data_db.put_block(scanned_block).map_err(Error::Wallet)?; + Ok(()) + }, + )?; Ok(()) } diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index 8eeb1b57e1..c449f98307 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -21,7 +21,7 @@ use zcash_primitives::{ zip32::{sapling::DiversifiableFullViewingKey, AccountId, Scope}, }; -use crate::data_api::{BlockMetadata, ScannedBlock}; +use crate::data_api::{BlockMetadata, ScannedBlock, ShieldedProtocol}; use crate::{ proto::compact_formats::CompactBlock, scan::{Batch, BatchRunner, Tasks}, @@ -116,17 +116,29 @@ pub enum ScanError { /// the current chain tip. PrevHashMismatch { at_height: BlockHeight }, - /// The block height field of the proposed new chain tip is not equal to the height of the - /// previous chain tip + 1. This variant stores a copy of the incorrect height value for - /// reporting purposes. + /// The block height field of the proposed new block is not equal to the height of the previous + /// block + 1. BlockHeightDiscontinuity { - previous_tip: BlockHeight, + prev_height: BlockHeight, new_height: BlockHeight, }, - /// The size of the Sapling note commitment tree was not provided as part of a [`CompactBlock`] - /// being scanned, making it impossible to construct the nullifier for a detected note. - SaplingTreeSizeUnknown { at_height: BlockHeight }, + /// The note commitment tree size for the given protocol at the proposed new block is not equal + /// to the size at the previous block plus the count of this block's outputs. + TreeSizeMismatch { + protocol: ShieldedProtocol, + at_height: BlockHeight, + given: u32, + computed: u32, + }, + + /// The size of the note commitment tree for the given protocol was not provided as part of a + /// [`CompactBlock`] being scanned, making it impossible to construct the nullifier for a + /// detected note. + TreeSizeUnknown { + protocol: ShieldedProtocol, + at_height: BlockHeight, + }, } impl fmt::Display for ScanError { @@ -137,11 +149,14 @@ impl fmt::Display for ScanError { "The parent hash of proposed block does not correspond to the block hash at height {}.", at_height ), - ScanError::BlockHeightDiscontinuity { previous_tip, new_height } => { - write!(f, "Block height discontinuity at height {}; next height is : {}", previous_tip, new_height) + ScanError::BlockHeightDiscontinuity { prev_height, new_height } => { + write!(f, "Block height discontinuity at height {}; next height is : {}", prev_height, new_height) } - ScanError::SaplingTreeSizeUnknown { at_height } => { - write!(f, "Unable to determine Sapling note commitment tree size at height {}", at_height) + ScanError::TreeSizeMismatch { protocol, at_height, given, computed } => { + write!(f, "The the {:?} note commitment tree size provided by a compact block did not match the expected size at height {}; given {}, expected {}", protocol, at_height, given, computed) + } + ScanError::TreeSizeUnknown { protocol, at_height } => { + write!(f, "Unable to determine {:?} note commitment tree size at height {}", protocol, at_height) } } } @@ -224,6 +239,46 @@ pub(crate) fn add_block_to_runner( } } +pub(crate) fn check_continuity( + block: &CompactBlock, + prior_block_metadata: Option<&BlockMetadata>, +) -> Option { + if let Some(prev) = prior_block_metadata { + if block.height() != prev.block_height() + 1 { + return Some(ScanError::BlockHeightDiscontinuity { + prev_height: prev.block_height(), + new_height: block.height(), + }); + } + + if block.prev_hash() != prev.block_hash() { + return Some(ScanError::PrevHashMismatch { + at_height: block.height(), + }); + } + + if let Some(given) = block + .chain_metadata + .as_ref() + .map(|m| m.sapling_commitment_tree_size) + { + let computed = prev.sapling_tree_size() + + u32::try_from(block.vtx.iter().map(|tx| tx.outputs.len()).sum::()) + .unwrap(); + if given != computed { + return Some(ScanError::TreeSizeMismatch { + protocol: ShieldedProtocol::Sapling, + at_height: block.height(), + given, + computed, + }); + } + } + } + + None +} + #[tracing::instrument(skip_all, fields(height = block.height))] pub(crate) fn scan_block_with_runner< P: consensus::Parameters + Send + 'static, @@ -237,26 +292,13 @@ pub(crate) fn scan_block_with_runner< prior_block_metadata: Option<&BlockMetadata>, mut batch_runner: Option<&mut TaggedBatchRunner>, ) -> Result, ScanError> { - let mut wtxs: Vec> = vec![]; - let mut sapling_note_commitments: Vec<(sapling::Node, Retention)> = vec![]; + if let Some(scan_error) = check_continuity(&block, prior_block_metadata) { + return Err(scan_error); + } + let cur_height = block.height(); let cur_hash = block.hash(); - if let Some(prev) = prior_block_metadata { - if cur_height != prev.block_height() + 1 { - return Err(ScanError::BlockHeightDiscontinuity { - previous_tip: prev.block_height(), - new_height: cur_height, - }); - } - - if block.prev_hash() != prev.block_hash() { - return Err(ScanError::PrevHashMismatch { - at_height: cur_height, - }); - } - } - // It's possible to make progress without a Sapling tree position if we don't have any Sapling // notes in the block, since we only use the position for constructing nullifiers for our own // received notes. Thus, we allow it to be optional here, and only produce an error if we try @@ -281,11 +323,14 @@ pub(crate) fn scan_block_with_runner< } }) .or_else(|| prior_block_metadata.map(|m| m.sapling_tree_size())) - .ok_or(ScanError::SaplingTreeSizeUnknown { + .ok_or(ScanError::TreeSizeUnknown { + protocol: ShieldedProtocol::Sapling, at_height: cur_height, })?; let compact_block_tx_count = block.vtx.len(); + let mut wtxs: Vec> = vec![]; + let mut sapling_note_commitments: Vec<(sapling::Node, Retention)> = vec![]; for (tx_idx, tx) in block.vtx.into_iter().enumerate() { let txid = tx.txid(); diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 478a3bf45b..46478fbc95 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -330,7 +330,14 @@ mod tests { insert_into_cache(&db_cache, &cb); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 1, + ) + .unwrap(); // Create a second fake CompactBlock sending more value to the address let (cb2, _) = fake_compact_block( @@ -344,7 +351,14 @@ mod tests { insert_into_cache(&db_cache, &cb2); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height() + 1, + 1, + ) + .unwrap(); } #[test] @@ -381,7 +395,14 @@ mod tests { insert_into_cache(&db_cache, &cb2); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 2, + ) + .unwrap(); // Create more fake CompactBlocks that don't connect to the scanned ones let (cb3, _) = fake_compact_block( @@ -405,7 +426,13 @@ mod tests { // Data+cache chain should be invalid at the data/cache boundary assert_matches!( - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None), + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height() + 2, + 2 + ), Err(_) // FIXME: check error result more closely ); } @@ -444,7 +471,14 @@ mod tests { insert_into_cache(&db_cache, &cb2); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 2, + ) + .unwrap(); // Create more fake CompactBlocks that contain a reorg let (cb3, _) = fake_compact_block( @@ -468,7 +502,13 @@ mod tests { // Data+cache chain should be invalid inside the cache assert_matches!( - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None), + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height() + 2, + 2 + ), Err(_) // FIXME: check error result more closely ); } @@ -516,7 +556,14 @@ mod tests { insert_into_cache(&db_cache, &cb2); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 2, + ) + .unwrap(); // Account balance should reflect both received notes assert_eq!( @@ -551,7 +598,14 @@ mod tests { ); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 2, + ) + .unwrap(); // Account balance should again reflect both received notes assert_eq!( @@ -586,7 +640,14 @@ mod tests { 0, ); insert_into_cache(&db_cache, &cb1); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 1, + ) + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -617,8 +678,8 @@ mod tests { &tests::network(), &db_cache, &mut db_data, - Some(sapling_activation_height() + 2), - None + sapling_activation_height() + 2, + 1 ), Ok(_) ); @@ -629,8 +690,8 @@ mod tests { &tests::network(), &db_cache, &mut db_data, - Some(sapling_activation_height() + 1), - Some(1), + sapling_activation_height() + 1, + 1, ) .unwrap(); assert_eq!( @@ -699,7 +760,14 @@ mod tests { insert_into_cache(&db_cache, &cb); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 1, + ) + .unwrap(); // Account balance should reflect the received note assert_eq!( @@ -720,7 +788,14 @@ mod tests { insert_into_cache(&db_cache, &cb2); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height() + 1, + 1, + ) + .unwrap(); // Account balance should reflect both received notes assert_eq!( @@ -761,7 +836,14 @@ mod tests { insert_into_cache(&db_cache, &cb); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 1, + ) + .unwrap(); // Account balance should reflect the received note assert_eq!( @@ -787,7 +869,14 @@ mod tests { ); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height() + 1, + 1, + ) + .unwrap(); // Account balance should equal the change assert_eq!( diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 0e14588941..d70fea0827 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -65,11 +65,9 @@ //! - `memo` the shielded memo associated with the output, if any. use rusqlite::{self, named_params, OptionalExtension, ToSql}; +use std::collections::HashMap; use std::convert::TryFrom; -use std::{ - collections::HashMap, - io::{self, Cursor}, -}; +use std::io::{self, Cursor}; use zcash_client_backend::data_api::ShieldedProtocol; use zcash_primitives::{ diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index a686ec6175..5108db19e2 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -571,7 +571,14 @@ pub(crate) mod tests { 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 1, + ) + .unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data @@ -598,7 +605,14 @@ pub(crate) mod tests { ) .0; insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height() + 1, + 1, + ) + .unwrap(); // Verified balance does not include the second note let (_, anchor_height2) = db_data @@ -651,7 +665,14 @@ pub(crate) mod tests { .0; insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height() + 2, + 8, + ) + .unwrap(); // Second spend still fails assert_matches!( @@ -681,11 +702,18 @@ pub(crate) mod tests { &dfvk, AddressType::DefaultExternal, value, - 11, + 10, ) .0; insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height() + 10, + 1, + ) + .unwrap(); // Second spend should now succeed assert_matches!( @@ -730,7 +758,14 @@ pub(crate) mod tests { 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 1, + ) + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -788,7 +823,14 @@ pub(crate) mod tests { .0; insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height() + 1, + 41, + ) + .unwrap(); // Second spend still fails assert_matches!( @@ -821,7 +863,14 @@ pub(crate) mod tests { ) .0; insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height() + 42, + 1, + ) + .unwrap(); // Second spend should now succeed create_spend_to_address( @@ -865,7 +914,14 @@ pub(crate) mod tests { 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 1, + ) + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -938,7 +994,14 @@ pub(crate) mod tests { .0; insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&network, &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &network, + &db_cache, + &mut db_data, + sapling_activation_height() + 1, + 42, + ) + .unwrap(); // Send the funds again, discarding history. // Neither transaction output is decryptable by the sender. @@ -971,7 +1034,14 @@ pub(crate) mod tests { 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 1, + ) + .unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data @@ -1030,7 +1100,14 @@ pub(crate) mod tests { 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 1, + ) + .unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data @@ -1103,7 +1180,14 @@ pub(crate) mod tests { insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 11, + ) + .unwrap(); // Verified balance matches total balance let total = Amount::from_u64(60000).unwrap(); @@ -1225,7 +1309,14 @@ pub(crate) mod tests { 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 1, + ) + .unwrap(); assert_matches!( shield_transparent_funds( From 81a32f284e4d411b04d98a1b12a844ed4be85bc0 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 6 Jul 2023 11:18:55 -0600 Subject: [PATCH 0051/1122] Apply suggestions from code review Co-authored-by: str4d --- zcash_client_backend/src/data_api/chain.rs | 14 ++++++++++++++ zcash_client_backend/src/scanning.rs | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 29e72b1a58..49d0f90051 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -51,6 +51,7 @@ //! ``` use zcash_primitives::{ + block::BlockHash, consensus::{self, BlockHeight}, sapling::{self, note_encryption::PreparedIncomingViewingKey}, zip32::Scope, @@ -197,6 +198,19 @@ where if let Some(scan_error) = check_continuity(&block, continuity_check_metadata.as_ref()) { return Err(Error::Scan(scan_error)); } + + if from_height == BlockHeight::from(0) { + // We can always derive a valid `continuity_check_metadata` for the + // genesis block, even if the block source doesn't have + // `sapling_commitment_tree_size`. So briefly set it to a dummy value that + // ensures the `map` below produces the correct genesis block value. + assert!(continuity_check_metadata.is_none()); + continuity_check_metadata = Some(BlockMetadata::from_parts( + BlockHeight::from(0), + BlockHash([0; 32]), + 0, + )); + } continuity_check_metadata = continuity_check_metadata.as_ref().map(|m| { BlockMetadata::from_parts( block.height(), diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index c449f98307..d2f419acd4 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -153,7 +153,7 @@ impl fmt::Display for ScanError { write!(f, "Block height discontinuity at height {}; next height is : {}", prev_height, new_height) } ScanError::TreeSizeMismatch { protocol, at_height, given, computed } => { - write!(f, "The the {:?} note commitment tree size provided by a compact block did not match the expected size at height {}; given {}, expected {}", protocol, at_height, given, computed) + write!(f, "The {:?} note commitment tree size provided by a compact block did not match the expected size at height {}; given {}, expected {}", protocol, at_height, given, computed) } ScanError::TreeSizeUnknown { protocol, at_height } => { write!(f, "Unable to determine {:?} note commitment tree size at height {}", protocol, at_height) From 1b5a24a65580fa17739e311060123329412a0a7a Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 5 Jul 2023 23:08:00 -0600 Subject: [PATCH 0052/1122] Batch data store writes of `put_block` Instead of calling `put_block` for each block scanned, `scan_cached_blocks` will now defer the block writes until the scan of a batch is complete and will perform the block writes and note commitment tree updates all within a single transaction. This should ordinarily be fine in terms of memory consumption, because the block data being saved is pruned to only that spend an output information that is related to transactions in the wallet, which will normally be sparse enough that the block range size that is appropriate for a given platform to run within a batch scanner adequately bounds the memory consumption of this pruned representation. --- zcash_client_backend/CHANGELOG.md | 6 +- zcash_client_backend/src/data_api.rs | 8 +- zcash_client_backend/src/data_api/chain.rs | 4 +- zcash_client_sqlite/src/lib.rs | 86 +++++++++++++--------- zcash_client_sqlite/src/wallet.rs | 4 +- 5 files changed, 62 insertions(+), 46 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 618bf2c887..02297d45ad 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -15,9 +15,9 @@ and this library adheres to Rust's notion of - `NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` - `ScannedBlock` - `ShieldedProtocol` - - `WalletRead::{block_metadata, block_fully_scanned, suggest_scan_ranges}` - - `WalletWrite::put_block` - `WalletCommitmentTrees` + - `WalletRead::{block_metadata, block_fully_scanned, suggest_scan_ranges}` + - `WalletWrite::put_blocks` - `chain::CommitmentTreeRoot` - `testing::MockWalletDb::new` - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}`: @@ -75,7 +75,7 @@ and this library adheres to Rust's notion of - `WalletRead::{get_commitment_tree, get_witnesses}` have been removed without replacement. The utility of these methods is now subsumed by those available from the `WalletCommitmentTrees` trait. - - `WalletWrite::advance_by_block` (use `WalletWrite::put_block` instead). + - `WalletWrite::advance_by_block` (use `WalletWrite::put_blocks` instead). - `PrunedBlock` has been replaced by `ScannedBlock` - `testing::MockWalletDb`, which is available under the `test-dependencies` feature flag, has been modified by the addition of a `sapling_tree` property. diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 216fc7174b..ceae06764a 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -494,9 +494,9 @@ pub trait WalletWrite: WalletRead { /// Updates the state of the wallet database by persisting the provided block information, /// along with the note commitments that were detected when scanning the block for transactions /// pertaining to this wallet. - fn put_block( + fn put_blocks( &mut self, - block: ScannedBlock, + block: Vec>, ) -> Result, Self::Error>; /// Caches a decrypted transaction in the persistent wallet store. @@ -771,9 +771,9 @@ pub mod testing { } #[allow(clippy::type_complexity)] - fn put_block( + fn put_blocks( &mut self, - _block: ScannedBlock, + _blocks: Vec>, ) -> Result, Self::Error> { Ok(vec![]) } diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 49d0f90051..7b45bb090e 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -237,6 +237,7 @@ where batch_runner.flush(); + let mut scanned_blocks = vec![]; block_source.with_blocks::<_, DbT::Error>( Some(from_height), Some(limit), @@ -265,11 +266,12 @@ where })); prior_block_metadata = Some(*scanned_block.metadata()); - data_db.put_block(scanned_block).map_err(Error::Wallet)?; + scanned_blocks.push(scanned_block); Ok(()) }, )?; + data_db.put_blocks(scanned_blocks).map_err(Error::Wallet)?; Ok(()) } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index c3d9b04bd5..4bb84e5966 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -394,53 +394,67 @@ impl WalletWrite for WalletDb ) } - #[tracing::instrument(skip_all, fields(height = u32::from(block.height())))] + #[tracing::instrument(skip_all, fields(height = blocks.first().map(|b| u32::from(b.height()))))] #[allow(clippy::type_complexity)] - fn put_block( + fn put_blocks( &mut self, - block: ScannedBlock, + blocks: Vec>, ) -> Result, Self::Error> { self.transactionally(|wdb| { - // Insert the block into the database. - wallet::put_block( - wdb.conn.0, - block.height(), - block.block_hash(), - block.block_time(), - block.metadata().sapling_tree_size(), - )?; - + let start_position = blocks.first().map(|block| { + Position::from( + u64::from(block.metadata().sapling_tree_size()) + - u64::try_from(block.sapling_commitments().len()).unwrap(), + ) + }); let mut wallet_note_ids = vec![]; - for tx in block.transactions() { - let tx_row = wallet::put_tx_meta(wdb.conn.0, tx, block.height())?; + let mut sapling_commitments = vec![]; + let mut end_height = None; - // Mark notes as spent and remove them from the scanning cache - for spend in &tx.sapling_spends { - wallet::sapling::mark_sapling_note_spent(wdb.conn.0, tx_row, spend.nf())?; - } + for block in blocks.into_iter() { + // Insert the block into the database. + wallet::put_block( + wdb.conn.0, + block.height(), + block.block_hash(), + block.block_time(), + block.metadata().sapling_tree_size(), + )?; - for output in &tx.sapling_outputs { - let received_note_id = - wallet::sapling::put_received_note(wdb.conn.0, output, tx_row)?; + for tx in block.transactions() { + let tx_row = wallet::put_tx_meta(wdb.conn.0, tx, block.height())?; - // Save witness for note. - wallet_note_ids.push(received_note_id); + // Mark notes as spent and remove them from the scanning cache + for spend in &tx.sapling_spends { + wallet::sapling::mark_sapling_note_spent(wdb.conn.0, tx_row, spend.nf())?; + } + + for output in &tx.sapling_outputs { + let received_note_id = + wallet::sapling::put_received_note(wdb.conn.0, output, tx_row)?; + + // Save witness for note. + wallet_note_ids.push(received_note_id); + } } + + end_height = Some(block.height()); + sapling_commitments.extend(block.into_sapling_commitments().into_iter()); } - let block_height = block.height(); - let sapling_tree_size = block.metadata().sapling_tree_size(); - let sapling_commitments_len = block.sapling_commitments().len(); - let mut sapling_commitments = block.into_sapling_commitments().into_iter(); - wdb.with_sapling_tree_mut::<_, _, SqliteClientError>(move |sapling_tree| { - let start_position = Position::from(u64::from(sapling_tree_size)) - - u64::try_from(sapling_commitments_len).unwrap(); - sapling_tree.batch_insert(start_position, &mut sapling_commitments)?; - Ok(()) - })?; - - // Update now-expired transactions that didn't get mined. - wallet::update_expired_notes(wdb.conn.0, block_height)?; + // We will have a start position and an end height in all cases where `blocks` is + // non-empty. + if let Some((start_position, end_height)) = start_position.zip(end_height) { + // Update the Sapling note commitment tree with all newly read note commitments + let mut sapling_commitments = sapling_commitments.into_iter(); + wdb.with_sapling_tree_mut::<_, _, SqliteClientError>(move |sapling_tree| { + sapling_tree.batch_insert(start_position, &mut sapling_commitments)?; + Ok(()) + })?; + + // Update now-expired transactions that didn't get mined. + wallet::update_expired_notes(wdb.conn.0, end_height)?; + } Ok(wallet_note_ids) }) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index d70fea0827..1f67e8d4d3 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1100,7 +1100,7 @@ pub(crate) fn put_legacy_transparent_utxo( /// as expired, up to the given block height. pub(crate) fn update_expired_notes( conn: &rusqlite::Connection, - height: BlockHeight, + expiry_height: BlockHeight, ) -> Result<(), SqliteClientError> { let mut stmt_update_expired = conn.prepare_cached( "UPDATE sapling_received_notes SET spent = NULL WHERE EXISTS ( @@ -1108,7 +1108,7 @@ pub(crate) fn update_expired_notes( WHERE id_tx = sapling_received_notes.spent AND block IS NULL AND expiry_height < ? )", )?; - stmt_update_expired.execute([u32::from(height)])?; + stmt_update_expired.execute([u32::from(expiry_height)])?; Ok(()) } From d55fa094644f989371275dd5480372141e2f9fed Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 6 Jul 2023 08:18:57 -0600 Subject: [PATCH 0053/1122] Add a check to ensure that blocks passed to `put_blocks` are sequential. --- zcash_client_backend/src/data_api.rs | 2 ++ zcash_client_sqlite/src/error.rs | 4 ++++ zcash_client_sqlite/src/lib.rs | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index ceae06764a..ad7a8bdda5 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -494,6 +494,8 @@ pub trait WalletWrite: WalletRead { /// Updates the state of the wallet database by persisting the provided block information, /// along with the note commitments that were detected when scanning the block for transactions /// pertaining to this wallet. + /// + /// `blocks` must be sequential, in order of increasing block height fn put_blocks( &mut self, block: Vec>, diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 1dd14f1c23..9058359d46 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -57,6 +57,9 @@ pub enum SqliteClientError { /// different hash. This indicates that a required rewind was not performed. BlockConflict(BlockHeight), + /// A range of blocks provided to the database as a unit was non-sequential + NonSequentialBlocks, + /// A requested rewind would violate invariants of the storage layer. The payload returned with /// this error is (safe rewind height, requested height). RequestedRewindInvalid(BlockHeight, BlockHeight), @@ -118,6 +121,7 @@ impl fmt::Display for SqliteClientError { SqliteClientError::Io(e) => write!(f, "{}", e), SqliteClientError::InvalidMemo(e) => write!(f, "{}", e), SqliteClientError::BlockConflict(h) => write!(f, "A block hash conflict occurred at height {}; rewind required.", u32::from(*h)), + SqliteClientError::NonSequentialBlocks => write!(f, "`put_blocks` requires that the provided block range be sequential"), SqliteClientError::DiversifierIndexOutOfRange => write!(f, "The space of available diversifier indices is exhausted"), SqliteClientError::KeyDerivationError(acct_id) => write!(f, "Key derivation failed for account {:?}", acct_id), SqliteClientError::AccountIdDiscontinuity => write!(f, "Wallet account identifiers must be sequential."), diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 4bb84e5966..1686145ad2 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -412,6 +412,10 @@ impl WalletWrite for WalletDb let mut end_height = None; for block in blocks.into_iter() { + if end_height.iter().any(|prev| block.height() != *prev + 1) { + return Err(SqliteClientError::NonSequentialBlocks); + } + // Insert the block into the database. wallet::put_block( wdb.conn.0, From a11545459b98b3687893636b805bcf6c09d09bbd Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 6 Jul 2023 22:00:48 +0000 Subject: [PATCH 0054/1122] Bump patched dependencies to bring in shardtree bugfixes --- Cargo.toml | 6 +++--- zcash_client_backend/src/data_api.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 073970f925..14e9d04239 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,6 @@ panic = 'abort' codegen-units = 1 [patch.crates-io] -incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "082109deacf8611ee7917732e19b56158bda96d5" } -shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "082109deacf8611ee7917732e19b56158bda96d5" } -orchard = { git = "https://github.com/zcash/orchard.git", rev = "5da41a6bbb44290e353ee4b38bcafe37ffe79ce8" } +incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "67111e29403c33f2e36d6924167f1d5f04ad0fc2" } +shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "67111e29403c33f2e36d6924167f1d5f04ad0fc2" } +orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index ad7a8bdda5..c62c3b57c4 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -568,7 +568,7 @@ pub trait WalletCommitmentTrees { pub mod testing { use incrementalmerkletree::Address; use secrecy::{ExposeSecret, SecretVec}; - use shardtree::{MemoryShardStore, ShardTree, ShardTreeError}; + use shardtree::{memory::MemoryShardStore, ShardTree, ShardTreeError}; use std::{collections::HashMap, convert::Infallible, ops::Range}; use zcash_primitives::{ From 8d792bb7b59df2da994c2547642a2f6055dbed4d Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 6 Jul 2023 21:54:48 +0000 Subject: [PATCH 0055/1122] zcash_client_sqlite: Fix `WalletDb::get_transaction` for unmined txs --- zcash_client_sqlite/CHANGELOG.md | 3 ++ zcash_client_sqlite/src/wallet.rs | 60 +++++++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 68ffd2ff31..ee38993f37 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -28,6 +28,9 @@ and this library adheres to Rust's notion of - Fixed an off-by-one error in the `BlockSource` implementation for the SQLite-backed `BlockDb` block database which could result in blocks being skipped at the start of scan ranges. +- `WalletDb::get_transaction` no longer returns an error when called on a transaction + that has not yet been mined, unless the transaction's consensus branch ID cannot be + determined by other means. ## [0.7.1] - 2023-05-17 diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 1f67e8d4d3..6f90151bbd 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -69,6 +69,7 @@ use std::collections::HashMap; use std::convert::TryFrom; use std::io::{self, Cursor}; use zcash_client_backend::data_api::ShieldedProtocol; +use zcash_primitives::transaction::TransactionData; use zcash_primitives::{ block::BlockHash, @@ -491,18 +492,65 @@ pub(crate) fn get_transaction( params: &P, id_tx: i64, ) -> Result { - let (tx_bytes, block_height): (Vec<_>, BlockHeight) = conn.query_row( - "SELECT raw, block FROM transactions + let (tx_bytes, block_height, expiry_height): ( + Vec<_>, + Option, + Option, + ) = conn.query_row( + "SELECT raw, block, expiry_height FROM transactions WHERE id_tx = ?", [id_tx], |row| { - let h: u32 = row.get(1)?; - Ok((row.get(0)?, BlockHeight::from(h))) + let h: Option = row.get(1)?; + let expiry: Option = row.get(2)?; + Ok(( + row.get(0)?, + h.map(BlockHeight::from), + expiry.map(BlockHeight::from), + )) }, )?; - Transaction::read(&tx_bytes[..], BranchId::for_height(params, block_height)) - .map_err(SqliteClientError::from) + // We need to provide a consensus branch ID so that pre-v5 `Transaction` structs + // (which don't commit directly to one) can store it internally. + // - If the transaction is mined, we use the block height to get the correct one. + // - If the transaction is unmined and has a cached non-zero expiry height, we use + // that (relying on the invariant that a transaction can't be mined across a network + // upgrade boundary, so the expiry height must be in the same epoch). + // - Otherwise, we use a placeholder for the initial transaction parse (as the + // consensus branch ID is not used there), and then either use its non-zero expiry + // height or return an error. + if let Some(height) = + block_height.or_else(|| expiry_height.filter(|h| h > &BlockHeight::from(0))) + { + Transaction::read(&tx_bytes[..], BranchId::for_height(params, height)) + .map_err(SqliteClientError::from) + } else { + let tx_data = Transaction::read(&tx_bytes[..], BranchId::Sprout) + .map_err(SqliteClientError::from)? + .into_data(); + + let expiry_height = tx_data.expiry_height(); + if expiry_height > BlockHeight::from(0) { + TransactionData::from_parts( + tx_data.version(), + BranchId::for_height(params, expiry_height), + tx_data.lock_time(), + expiry_height, + tx_data.transparent_bundle().cloned(), + tx_data.sprout_bundle().cloned(), + tx_data.sapling_bundle().cloned(), + tx_data.orchard_bundle().cloned(), + ) + .freeze() + .map_err(SqliteClientError::from) + } else { + Err(SqliteClientError::CorruptedData( + "Consensus branch ID not known, cannot parse this transaction until it is mined" + .to_string(), + )) + } + } } /// Returns the memo for a sent note. From 6fa0b46d8eca525d3174a2605519c1687e0c88fa Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 6 Jul 2023 08:37:28 -0600 Subject: [PATCH 0056/1122] Implement `suggest_scan_ranges` and `update_chain_tip` This implements a priority queue backed by the wallet database for scan range ordering. The scan queue is updated on each call to `put_blocks` or to `update_chain_tip`. --- zcash_client_backend/CHANGELOG.md | 14 +- zcash_client_backend/proto/service.proto | 24 + zcash_client_backend/src/data_api.rs | 37 +- zcash_client_backend/src/data_api/chain.rs | 132 ++- zcash_client_backend/src/data_api/scanning.rs | 107 +++ zcash_client_backend/src/proto/service.rs | 148 +++ zcash_client_backend/src/scanning.rs | 35 +- zcash_client_sqlite/src/chain.rs | 144 +-- zcash_client_sqlite/src/lib.rs | 60 +- zcash_client_sqlite/src/wallet.rs | 34 +- zcash_client_sqlite/src/wallet/init.rs | 10 + .../init/migrations/shardtree_support.rs | 28 +- zcash_client_sqlite/src/wallet/scanning.rs | 905 ++++++++++++++++++ zcash_primitives/CHANGELOG.md | 1 + zcash_primitives/src/consensus.rs | 6 + 15 files changed, 1517 insertions(+), 168 deletions(-) create mode 100644 zcash_client_backend/src/data_api/scanning.rs create mode 100644 zcash_client_sqlite/src/wallet/scanning.rs diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 02297d45ad..e5c1ae7d77 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -17,8 +17,9 @@ and this library adheres to Rust's notion of - `ShieldedProtocol` - `WalletCommitmentTrees` - `WalletRead::{block_metadata, block_fully_scanned, suggest_scan_ranges}` - - `WalletWrite::put_blocks` + - `WalletWrite::{put_blocks, update_chain_tip}` - `chain::CommitmentTreeRoot` + - `scanning` A new module containing types required for `suggest_scan_ranges` - `testing::MockWalletDb::new` - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}`: - `zcash_client_backend::wallet::WalletSaplingOutput::note_commitment_tree_position` @@ -36,8 +37,10 @@ and this library adheres to Rust's notion of and its signature has changed; it now subsumes the removed `WalletRead::get_all_nullifiers`. - `WalletRead::get_target_and_anchor_heights` now takes its argument as a `NonZeroU32` - `chain::scan_cached_blocks` now takes a `from_height` argument that - permits the caller to control the starting position of the scan range - In addition, the `limit` parameter is now required. + permits the caller to control the starting position of the scan range. + In addition, the `limit` parameter is now required and has type `usize`. + - `chain::BlockSource::with_blocks` now takes its limit as an `Option` + instead of `Option`. - A new `CommitmentTree` variant has been added to `data_api::error::Error` - `data_api::wallet::{create_spend_to_address, create_proposed_transaction, shield_transparent_funds}` all now require that `WalletCommitmentTrees` be @@ -67,7 +70,8 @@ and this library adheres to Rust's notion of method now takes an optional `BlockMetadata` argument instead of a base commitment tree and incremental witnesses for each previously-known note. In addition, the return type has now been updated to return a `Result`. - +- `proto/service.proto` has been updated to include the new GRPC endpoints + supported by lightwalletd v0.4.15 ### Removed - `zcash_client_backend::data_api`: @@ -81,8 +85,6 @@ and this library adheres to Rust's notion of feature flag, has been modified by the addition of a `sapling_tree` property. - `wallet::input_selection`: - `Proposal::target_height` (use `Proposal::min_target_height` instead). -- `zcash_client_backend::data_api::chain::validate_chain` TODO: document how - to handle validation given out-of-order blocks. - `zcash_client_backend::data_api::chain::error::{ChainError, Cause}` have been replaced by `zcash_client_backend::scanning::ScanError` - `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}` diff --git a/zcash_client_backend/proto/service.proto b/zcash_client_backend/proto/service.proto index d7f11dcd69..0945661478 100644 --- a/zcash_client_backend/proto/service.proto +++ b/zcash_client_backend/proto/service.proto @@ -118,6 +118,22 @@ message TreeState { string orchardTree = 6; // orchard commitment tree state } +enum ShieldedProtocol { + sapling = 0; + orchard = 1; +} + +message GetSubtreeRootsArg { + uint32 startIndex = 1; // Index identifying where to start returning subtree roots + ShieldedProtocol shieldedProtocol = 2; // Shielded protocol to return subtree roots for + uint32 maxEntries = 3; // Maximum number of entries to return, or 0 for all entries. +} +message SubtreeRoot { + bytes rootHash = 2; // The 32-byte Merkle root of the subtree. + bytes completingBlockHash = 3; // The hash of the block that completed this subtree. + uint64 completingBlockHeight = 4; // The height of the block that completed this subtree in the main chain. +} + // Results are sorted by height, which makes it easy to issue another // request that picks up from where the previous left off. message GetAddressUtxosArg { @@ -142,8 +158,12 @@ service CompactTxStreamer { rpc GetLatestBlock(ChainSpec) returns (BlockID) {} // Return the compact block corresponding to the given block identifier rpc GetBlock(BlockID) returns (CompactBlock) {} + // Same as GetBlock except actions contain only nullifiers + rpc GetBlockNullifiers(BlockID) returns (CompactBlock) {} // Return a list of consecutive compact blocks rpc GetBlockRange(BlockRange) returns (stream CompactBlock) {} + // Same as GetBlockRange except actions contain only nullifiers + rpc GetBlockRangeNullifiers(BlockRange) returns (stream CompactBlock) {} // Return the requested full (not compact) transaction (as from zcashd) rpc GetTransaction(TxFilter) returns (RawTransaction) {} @@ -177,6 +197,10 @@ service CompactTxStreamer { rpc GetTreeState(BlockID) returns (TreeState) {} rpc GetLatestTreeState(Empty) returns (TreeState) {} + // Returns a stream of information about roots of subtrees of the Sapling and Orchard + // note commitment trees. + rpc GetSubtreeRoots(GetSubtreeRootsArg) returns (stream SubtreeRoot) {} + rpc GetAddressUtxos(GetAddressUtxosArg) returns (GetAddressUtxosReplyList) {} rpc GetAddressUtxosStream(GetAddressUtxosArg) returns (stream GetAddressUtxosReply) {} diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index c62c3b57c4..4f4c80c869 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -1,9 +1,9 @@ //! Interfaces for wallet data persistence & low-level wallet utilities. +use std::cmp; use std::collections::HashMap; use std::fmt::Debug; use std::num::NonZeroU32; -use std::{cmp, ops::Range}; use incrementalmerkletree::Retention; use secrecy::SecretVec; @@ -29,9 +29,11 @@ use crate::{ }; use self::chain::CommitmentTreeRoot; +use self::scanning::ScanRange; pub mod chain; pub mod error; +pub mod scanning; pub mod wallet; pub const SAPLING_SHARD_HEIGHT: u8 = sapling::NOTE_COMMITMENT_TREE_DEPTH / 2; @@ -88,11 +90,7 @@ pub trait WalletRead { /// to the wallet are detected. /// /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock - fn suggest_scan_ranges( - &self, - batch_size: usize, - limit: usize, - ) -> Result>, Self::Error>; + fn suggest_scan_ranges(&self) -> Result, Self::Error>; /// Returns the default target height (for the block in which a new /// transaction would be mined) and anchor height (to use for a new @@ -501,6 +499,15 @@ pub trait WalletWrite: WalletRead { block: Vec>, ) -> Result, Self::Error>; + /// Updates the wallet's view of the blockchain. + /// + /// This method is used to provide the wallet with information about the state of the + /// blockchain, and detect any previously scanned that needs to be re-validated before + /// proceeding with scanning. It should be called at wallet startup prior to calling + /// [`WalletRead::suggest_scan_ranges`] in order to provide the wallet with the information it + /// needs to correctly prioritize scanning operations. + fn update_chain_tip(&mut self, tip_height: BlockHeight) -> Result<(), Self::Error>; + /// Caches a decrypted transaction in the persistent wallet store. fn store_decrypted_tx( &mut self, @@ -569,7 +576,7 @@ pub mod testing { use incrementalmerkletree::Address; use secrecy::{ExposeSecret, SecretVec}; use shardtree::{memory::MemoryShardStore, ShardTree, ShardTreeError}; - use std::{collections::HashMap, convert::Infallible, ops::Range}; + use std::{collections::HashMap, convert::Infallible}; use zcash_primitives::{ block::BlockHash, @@ -591,9 +598,9 @@ pub mod testing { }; use super::{ - chain::CommitmentTreeRoot, BlockMetadata, DecryptedTransaction, NullifierQuery, - ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, WalletWrite, - SAPLING_SHARD_HEIGHT, + chain::CommitmentTreeRoot, scanning::ScanRange, BlockMetadata, DecryptedTransaction, + NullifierQuery, ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, + WalletWrite, SAPLING_SHARD_HEIGHT, }; pub struct MockWalletDb { @@ -634,11 +641,7 @@ pub mod testing { Ok(None) } - fn suggest_scan_ranges( - &self, - _batch_size: usize, - _limit: usize, - ) -> Result>, Self::Error> { + fn suggest_scan_ranges(&self) -> Result, Self::Error> { Ok(vec![]) } @@ -780,6 +783,10 @@ pub mod testing { Ok(vec![]) } + fn update_chain_tip(&mut self, _tip_height: BlockHeight) -> Result<(), Self::Error> { + Ok(()) + } + fn store_decrypted_tx( &mut self, _received_tx: DecryptedTransaction, diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 7b45bb090e..54a285eb44 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -7,18 +7,21 @@ //! # #[cfg(feature = "test-dependencies")] //! # { //! use zcash_primitives::{ -//! consensus::{BlockHeight, Network, Parameters} +//! consensus::{BlockHeight, Network, Parameters}, +//! sapling //! }; //! //! use zcash_client_backend::{ //! data_api::{ -//! WalletRead, WalletWrite, +//! WalletRead, WalletWrite, WalletCommitmentTrees, //! chain::{ //! BlockSource, +//! CommitmentTreeRoot, //! error::Error, //! scan_cached_blocks, //! testing as chain_testing, //! }, +//! scanning::ScanPriority, //! testing, //! }, //! }; @@ -32,20 +35,109 @@ //! # fn test() -> Result<(), Error<(), Infallible>> { //! let network = Network::TestNetwork; //! let block_source = chain_testing::MockBlockSource; -//! let mut db_data = testing::MockWalletDb::new(Network::TestNetwork); +//! let mut wallet_db = testing::MockWalletDb::new(Network::TestNetwork); //! -//! // 1) Download new CompactBlocks into block_source. -//! // -//! // 2) FIXME: Obtain necessary block metadata for continuity checking? -//! // -//! // 3) Scan cached blocks. -//! // -//! // FIXME: update documentation on how to detect when a rewind is required. -//! // -//! // At this point, the cache and scanned data are locally consistent (though not -//! // necessarily consistent with the latest chain tip - this would be discovered the -//! // next time this codepath is executed after new blocks are received). -//! scan_cached_blocks(&network, &block_source, &mut db_data, BlockHeight::from(0), 10) +//! // 1) Download note commitment tree data from lightwalletd +//! let roots: Vec> = unimplemented!(); +//! +//! // 2) Pass the commitment tree data to the database. +//! wallet_db.put_sapling_subtree_roots(0, &roots).unwrap(); +//! +//! // 3) Download chain tip metadata from lightwalletd +//! let tip_height: BlockHeight = unimplemented!(); +//! +//! // 4) Notify the wallet of the updated chain tip. +//! wallet_db.update_chain_tip(tip_height).map_err(Error::Wallet)?; +//! +//! // 5) Get the suggested scan ranges from the wallet database +//! let mut scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?; +//! +//! // 6) Run the following loop until the wallet's view of the chain tip as of the previous wallet +//! // session is valid. +//! loop { +//! // If there is a range of blocks that needs to be verified, it will always be returned as +//! // the first element of the vector of suggested ranges. +//! match scan_ranges.first() { +//! Some(scan_range) if scan_range.priority() == ScanPriority::Verify => { +//! // Download the blocks in `scan_range` into the block source, overwriting any +//! // existing blocks in this range. +//! unimplemented!(); +//! +//! // Scan the downloaded blocks +//! let scan_result = scan_cached_blocks( +//! &network, +//! &block_source, +//! &mut wallet_db, +//! scan_range.block_range().start, +//! scan_range.len() +//! ); +//! +//! // Check for scanning errors that indicate that the wallet's chain tip is out of +//! // sync with blockchain history. +//! match scan_result { +//! Ok(_) => { +//! // At this point, the cache and scanned data are locally consistent (though +//! // not necessarily consistent with the latest chain tip - this would be +//! // discovered the next time this codepath is executed after new blocks are +//! // received) so we can break out of the loop. +//! break; +//! } +//! Err(Error::Scan(err)) if err.is_continuity_error() => { +//! // Pick a height to rewind to, which must be at least one block before +//! // the height at which the error occurred, but may be an earlier height +//! // determined based on heuristics such as the platform, available bandwidth, +//! // size of recent CompactBlocks, etc. +//! let rewind_height = err.at_height().saturating_sub(10); +//! +//! // Rewind to the chosen height. +//! wallet_db.truncate_to_height(rewind_height).map_err(Error::Wallet)?; +//! +//! // Delete cached blocks from rewind_height onwards. +//! // +//! // This does imply that assumed-valid blocks will be re-downloaded, but it +//! // is also possible that in the intervening time, a chain reorg has +//! // occurred that orphaned some of those blocks. +//! unimplemented!(); +//! } +//! Err(other) => { +//! // Handle or return other errors +//! } +//! } +//! +//! // Truncation will have updated the suggested scan ranges, so we now +//! // re_request +//! scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?; +//! } +//! _ => { +//! // Nothing to verify; break out of the loop +//! break; +//! } +//! } +//! } +//! +//! // 7) Loop over the remaining suggested scan ranges, retrieving the requested data and calling +//! // `scan_cached_blocks` on each range. Periodically, or if a continuity error is +//! // encountered, this process should be repeated starting at step (3). +//! let scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?; +//! for scan_range in scan_ranges { +//! // Download the blocks in `scan_range` into the block source. While in this example this +//! // step is performed in-line, it's fine for the download of scan ranges to be asynchronous +//! // and for the scanner to process the downloaded ranges as they become available in a +//! // separate thread. +//! unimplemented!(); +//! +//! // Scan the downloaded blocks, +//! let scan_result = scan_cached_blocks( +//! &network, +//! &block_source, +//! &mut wallet_db, +//! scan_range.block_range().start, +//! scan_range.len() +//! )?; +//! +//! // Handle scan errors, etc. +//! } +//! # Ok(()) //! # } //! # } //! ``` @@ -58,14 +150,12 @@ use zcash_primitives::{ }; use crate::{ - data_api::{NullifierQuery, WalletWrite}, + data_api::{BlockMetadata, NullifierQuery, WalletWrite}, proto::compact_formats::CompactBlock, scan::BatchRunner, scanning::{add_block_to_runner, check_continuity, scan_block_with_runner}, }; -use super::BlockMetadata; - pub mod error; use error::Error; @@ -114,7 +204,7 @@ pub trait BlockSource { fn with_blocks( &self, from_height: Option, - limit: Option, + limit: Option, with_row: F, ) -> Result<(), error::Error> where @@ -145,7 +235,7 @@ pub fn scan_cached_blocks( block_source: &BlockSourceT, data_db: &mut DbT, from_height: BlockHeight, - limit: u32, + limit: usize, ) -> Result<(), Error> where ParamsT: consensus::Parameters + Send + 'static, @@ -292,7 +382,7 @@ pub mod testing { fn with_blocks( &self, _from_height: Option, - _limit: Option, + _limit: Option, _with_row: F, ) -> Result<(), Error> where diff --git a/zcash_client_backend/src/data_api/scanning.rs b/zcash_client_backend/src/data_api/scanning.rs new file mode 100644 index 0000000000..6a48297807 --- /dev/null +++ b/zcash_client_backend/src/data_api/scanning.rs @@ -0,0 +1,107 @@ +use std::ops::Range; + +use zcash_primitives::consensus::BlockHeight; + +/// Scanning range priority levels. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum ScanPriority { + /// Block ranges that have already been scanned have lowest priority. + Scanned, + /// Block ranges to be scanned to advance the fully-scanned height. + Historic, + /// Block ranges adjacent to wallet open heights. + OpenAdjacent, + /// Blocks that must be scanned to complete note commitment tree shards adjacent to found notes. + FoundNote, + /// Blocks that must be scanned to complete the latest note commitment tree shard. + ChainTip, + /// A previously-scanned range that must be verified has highest priority. + Verify, +} + +/// A range of blocks to be scanned, along with its associated priority. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScanRange { + block_range: Range, + priority: ScanPriority, +} + +impl ScanRange { + /// Constructs a scan range from its constituent parts. + pub fn from_parts(block_range: Range, priority: ScanPriority) -> Self { + assert!(block_range.end >= block_range.start); + ScanRange { + block_range, + priority, + } + } + + /// Returns the range of block heights to be scanned. + pub fn block_range(&self) -> &Range { + &self.block_range + } + + /// Returns the priority with which the scan range should be scanned. + pub fn priority(&self) -> ScanPriority { + self.priority + } + + /// Returns whether or not the scan range is empty. + pub fn is_empty(&self) -> bool { + self.block_range.end == self.block_range.start + } + + /// Returns the number of blocks in the scan range. + pub fn len(&self) -> usize { + usize::try_from(u32::from(self.block_range.end) - u32::from(self.block_range.start)) + .unwrap() + } + + /// Shifts the start of the block range to the right if `block_height > + /// self.block_range().start`. Returns `None` if the resulting range would + /// be empty. + pub fn truncate_start(&self, block_height: BlockHeight) -> Option { + if block_height >= self.block_range.end { + None + } else { + Some(ScanRange { + block_range: block_height..self.block_range.end, + priority: self.priority, + }) + } + } + + /// Shifts the end of the block range to the left if `block_height < + /// self.block_range().end`. Returns `None` if the resulting range would + /// be empty. + pub fn truncate_end(&self, block_height: BlockHeight) -> Option { + if block_height <= self.block_range.start { + None + } else { + Some(ScanRange { + block_range: self.block_range.start..block_height, + priority: self.priority, + }) + } + } + + /// Splits this scan range at the specified height, such that the provided height becomes the + /// end of the first range returned and the start of the second. Returns `None` if + /// `p <= self.block_range().start || p >= self.block_range().end`. + pub fn split_at(&self, p: BlockHeight) -> Option<(Self, Self)> { + if p > self.block_range.start && p < self.block_range.end { + Some(( + ScanRange { + block_range: self.block_range.start..p, + priority: self.priority, + }, + ScanRange { + block_range: p..self.block_range.end, + priority: self.priority, + }, + )) + } else { + None + } + } +} diff --git a/zcash_client_backend/src/proto/service.rs b/zcash_client_backend/src/proto/service.rs index 38b15abdbf..677e43e307 100644 --- a/zcash_client_backend/src/proto/service.rs +++ b/zcash_client_backend/src/proto/service.rs @@ -187,6 +187,32 @@ pub struct TreeState { #[prost(string, tag = "6")] pub orchard_tree: ::prost::alloc::string::String, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetSubtreeRootsArg { + /// Index identifying where to start returning subtree roots + #[prost(uint32, tag = "1")] + pub start_index: u32, + /// Shielded protocol to return subtree roots for + #[prost(enumeration = "ShieldedProtocol", tag = "2")] + pub shielded_protocol: i32, + /// Maximum number of entries to return, or 0 for all entries. + #[prost(uint32, tag = "3")] + pub max_entries: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SubtreeRoot { + /// The 32-byte Merkle root of the subtree. + #[prost(bytes = "vec", tag = "2")] + pub root_hash: ::prost::alloc::vec::Vec, + /// The hash of the block that completed this subtree. + #[prost(bytes = "vec", tag = "3")] + pub completing_block_hash: ::prost::alloc::vec::Vec, + /// The height of the block that completed this subtree in the main chain. + #[prost(uint64, tag = "4")] + pub completing_block_height: u64, +} /// Results are sorted by height, which makes it easy to issue another /// request that picks up from where the previous left off. #[allow(clippy::derive_partial_eq_without_eq)] @@ -222,6 +248,32 @@ pub struct GetAddressUtxosReplyList { #[prost(message, repeated, tag = "1")] pub address_utxos: ::prost::alloc::vec::Vec, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ShieldedProtocol { + Sapling = 0, + Orchard = 1, +} +impl ShieldedProtocol { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ShieldedProtocol::Sapling => "sapling", + ShieldedProtocol::Orchard => "orchard", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "sapling" => Some(Self::Sapling), + "orchard" => Some(Self::Orchard), + _ => None, + } + } +} /// Generated client implementations. pub mod compact_tx_streamer_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] @@ -366,6 +418,37 @@ pub mod compact_tx_streamer_client { ); self.inner.unary(req, path, codec).await } + /// Same as GetBlock except actions contain only nullifiers + pub async fn get_block_nullifiers( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/cash.z.wallet.sdk.rpc.CompactTxStreamer/GetBlockNullifiers", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "cash.z.wallet.sdk.rpc.CompactTxStreamer", + "GetBlockNullifiers", + ), + ); + self.inner.unary(req, path, codec).await + } /// Return a list of consecutive compact blocks pub async fn get_block_range( &mut self, @@ -399,6 +482,39 @@ pub mod compact_tx_streamer_client { ); self.inner.server_streaming(req, path, codec).await } + /// Same as GetBlockRange except actions contain only nullifiers + pub async fn get_block_range_nullifiers( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response< + tonic::codec::Streaming, + >, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/cash.z.wallet.sdk.rpc.CompactTxStreamer/GetBlockRangeNullifiers", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "cash.z.wallet.sdk.rpc.CompactTxStreamer", + "GetBlockRangeNullifiers", + ), + ); + self.inner.server_streaming(req, path, codec).await + } /// Return the requested full (not compact) transaction (as from zcashd) pub async fn get_transaction( &mut self, @@ -671,6 +787,38 @@ pub mod compact_tx_streamer_client { ); self.inner.unary(req, path, codec).await } + /// Returns a stream of information about roots of subtrees of the Sapling and Orchard + /// note commitment trees. + pub async fn get_subtree_roots( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/cash.z.wallet.sdk.rpc.CompactTxStreamer/GetSubtreeRoots", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "cash.z.wallet.sdk.rpc.CompactTxStreamer", + "GetSubtreeRoots", + ), + ); + self.inner.server_streaming(req, path, codec).await + } pub async fn get_address_utxos( &mut self, request: impl tonic::IntoRequest, diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index d2f419acd4..e49f71331c 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -110,7 +110,7 @@ impl ScanningKey for SaplingIvk { } /// Errors that may occur in chain scanning -#[derive(Copy, Clone, Debug)] +#[derive(Clone, Debug)] pub enum ScanError { /// The hash of the parent block given by a proposed new chain tip does not match the hash of /// the current chain tip. @@ -141,21 +141,46 @@ pub enum ScanError { }, } +impl ScanError { + /// Returns whether this error is the result of a failed continuity check + pub fn is_continuity_error(&self) -> bool { + use ScanError::*; + match self { + PrevHashMismatch { .. } => true, + BlockHeightDiscontinuity { .. } => true, + TreeSizeMismatch { .. } => true, + TreeSizeUnknown { .. } => false, + } + } + + /// Returns the block height at which the scan error occurred + pub fn at_height(&self) -> BlockHeight { + use ScanError::*; + match self { + PrevHashMismatch { at_height } => *at_height, + BlockHeightDiscontinuity { new_height, .. } => *new_height, + TreeSizeMismatch { at_height, .. } => *at_height, + TreeSizeUnknown { at_height, .. } => *at_height, + } + } +} + impl fmt::Display for ScanError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use ScanError::*; match &self { - ScanError::PrevHashMismatch { at_height } => write!( + PrevHashMismatch { at_height } => write!( f, "The parent hash of proposed block does not correspond to the block hash at height {}.", at_height ), - ScanError::BlockHeightDiscontinuity { prev_height, new_height } => { + BlockHeightDiscontinuity { prev_height, new_height } => { write!(f, "Block height discontinuity at height {}; next height is : {}", prev_height, new_height) } - ScanError::TreeSizeMismatch { protocol, at_height, given, computed } => { + TreeSizeMismatch { protocol, at_height, given, computed } => { write!(f, "The {:?} note commitment tree size provided by a compact block did not match the expected size at height {}; given {}, expected {}", protocol, at_height, given, computed) } - ScanError::TreeSizeUnknown { protocol, at_height } => { + TreeSizeUnknown { protocol, at_height } => { write!(f, "Unable to determine {:?} note commitment tree size at height {}", protocol, at_height) } } diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 46478fbc95..9091899754 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -26,10 +26,14 @@ pub mod migrations; /// Starting at `from_height`, the `with_row` callback is invoked with each block retrieved from /// the backing store. If the `limit` value provided is `None`, all blocks are traversed up to the /// maximum height. +/// +/// # Panics +/// +/// Panics if the provided `limit` value exceeds the range of a u32 pub(crate) fn blockdb_with_blocks( block_source: &BlockDb, from_height: Option, - limit: Option, + limit: Option, mut with_row: F, ) -> Result<(), Error> where @@ -52,7 +56,9 @@ where let mut rows = stmt_blocks .query(params![ from_height.map_or(0u32, u32::from), - limit.unwrap_or(u32::max_value()), + limit + .and_then(|l| u32::try_from(l).ok()) + .unwrap_or(u32::MAX) ]) .map_err(to_chain_error)?; @@ -194,11 +200,15 @@ pub(crate) fn blockmetadb_find_block( /// Starting at `from_height`, the `with_row` callback is invoked with each block retrieved from /// the backing store. If the `limit` value provided is `None`, all blocks are traversed up to the /// maximum height for which metadata is available. +/// +/// # Panics +/// +/// Panics if the provided `limit` value exceeds the range of a u32 #[cfg(feature = "unstable")] pub(crate) fn fsblockdb_with_blocks( cache: &FsBlockDb, from_height: Option, - limit: Option, + limit: Option, mut with_block: F, ) -> Result<(), Error> where @@ -223,7 +233,9 @@ where .query_map( params![ from_height.map_or(0u32, u32::from), - limit.unwrap_or(u32::max_value()), + limit + .and_then(|l| u32::try_from(l).ok()) + .unwrap_or(u32::MAX) ], |row| { Ok(BlockMeta { @@ -279,11 +291,12 @@ mod tests { use zcash_client_backend::{ address::RecipientAddress, data_api::{ - chain::scan_cached_blocks, + chain::{error::Error, scan_cached_blocks}, wallet::{input_selection::GreedyInputSelector, spend}, WalletRead, WalletWrite, }, fees::{zip317::SingleOutputChangeStrategy, DustOutputPolicy}, + scanning::ScanError, wallet::OvkPolicy, zip321::{Payment, TransactionRequest}, }; @@ -315,12 +328,9 @@ mod tests { assert_matches!(db_data.get_max_height_hash(), Ok(None)); // Create a fake CompactBlock sending value to the address - let fake_block_hash = BlockHash([0; 32]); - let fake_block_height = sapling_activation_height(); - let (cb, _) = fake_compact_block( - fake_block_height, - fake_block_hash, + sapling_activation_height(), + BlockHash([0; 32]), &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), @@ -348,97 +358,24 @@ mod tests { Amount::from_u64(7).unwrap(), 1, ); - insert_into_cache(&db_cache, &cb2); - - // Scan the cache again - scan_cached_blocks( - &tests::network(), - &db_cache, - &mut db_data, - sapling_activation_height() + 1, - 1, - ) - .unwrap(); - } - - #[test] - fn invalid_chain_cache_disconnected() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); - init_cache_database(&db_cache).unwrap(); - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); - - // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); - - // Create some fake CompactBlocks - let (cb, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), - &dfvk, - AddressType::DefaultExternal, - Amount::from_u64(5).unwrap(), - 0, - ); - let (cb2, _) = fake_compact_block( - sapling_activation_height() + 1, - cb.hash(), - &dfvk, - AddressType::DefaultExternal, - Amount::from_u64(7).unwrap(), - 1, - ); - insert_into_cache(&db_cache, &cb); insert_into_cache(&db_cache, &cb2); - // Scan the cache - scan_cached_blocks( - &tests::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 2, - ) - .unwrap(); - - // Create more fake CompactBlocks that don't connect to the scanned ones - let (cb3, _) = fake_compact_block( - sapling_activation_height() + 2, - BlockHash([1; 32]), - &dfvk, - AddressType::DefaultExternal, - Amount::from_u64(8).unwrap(), - 2, - ); - let (cb4, _) = fake_compact_block( - sapling_activation_height() + 3, - cb3.hash(), - &dfvk, - AddressType::DefaultExternal, - Amount::from_u64(3).unwrap(), - 3, - ); - insert_into_cache(&db_cache, &cb3); - insert_into_cache(&db_cache, &cb4); - - // Data+cache chain should be invalid at the data/cache boundary + // Scanning should detect no inconsistencies assert_matches!( scan_cached_blocks( &tests::network(), &db_cache, &mut db_data, - sapling_activation_height() + 2, - 2 + sapling_activation_height() + 1, + 1, ), - Err(_) // FIXME: check error result more closely + Ok(()) ); } #[test] - fn invalid_chain_cache_reorg() { + fn invalid_chain_cache_disconnected() { let cache_file = NamedTempFile::new().unwrap(); let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); init_cache_database(&db_cache).unwrap(); @@ -470,20 +407,22 @@ mod tests { insert_into_cache(&db_cache, &cb); insert_into_cache(&db_cache, &cb2); - // Scan the cache - scan_cached_blocks( - &tests::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 2, - ) - .unwrap(); + // Scanning the cache should find no inconsistencies + assert_matches!( + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 2, + ), + Ok(()) + ); - // Create more fake CompactBlocks that contain a reorg + // Create more fake CompactBlocks that don't connect to the scanned ones let (cb3, _) = fake_compact_block( sapling_activation_height() + 2, - cb2.hash(), + BlockHash([1; 32]), &dfvk, AddressType::DefaultExternal, Amount::from_u64(8).unwrap(), @@ -491,7 +430,7 @@ mod tests { ); let (cb4, _) = fake_compact_block( sapling_activation_height() + 3, - BlockHash([1; 32]), + cb3.hash(), &dfvk, AddressType::DefaultExternal, Amount::from_u64(3).unwrap(), @@ -500,7 +439,7 @@ mod tests { insert_into_cache(&db_cache, &cb3); insert_into_cache(&db_cache, &cb4); - // Data+cache chain should be invalid inside the cache + // Data+cache chain should be invalid at the data/cache boundary assert_matches!( scan_cached_blocks( &tests::network(), @@ -509,7 +448,8 @@ mod tests { sapling_activation_height() + 2, 2 ), - Err(_) // FIXME: check error result more closely + Err(Error::Scan(ScanError::PrevHashMismatch { at_height })) + if at_height == sapling_activation_height() + 2 ); } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 1686145ad2..57aa166c2b 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -36,7 +36,6 @@ use either::Either; use rusqlite::{self, Connection}; use secrecy::{ExposeSecret, SecretVec}; use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, io, ops::Range, path::Path}; -use wallet::commitment_tree::put_shard_roots; use incrementalmerkletree::Position; use shardtree::{ShardTree, ShardTreeError}; @@ -58,6 +57,7 @@ use zcash_client_backend::{ data_api::{ self, chain::{BlockSource, CommitmentTreeRoot}, + scanning::ScanRange, BlockMetadata, DecryptedTransaction, NullifierQuery, PoolType, Recipient, ScannedBlock, SentTransaction, ShieldedProtocol, WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, @@ -80,13 +80,18 @@ use { pub mod chain; pub mod error; pub mod serialization; + pub mod wallet; +use wallet::commitment_tree::put_shard_roots; /// The maximum number of blocks the wallet is allowed to rewind. This is /// consistent with the bound in zcashd, and allows block data deeper than /// this delta from the chain tip to be pruned. pub(crate) const PRUNING_DEPTH: u32 = 100; +/// The number of blocks to re-verify when the chain tip is updated. +pub(crate) const VALIDATION_DEPTH: u32 = 10; + pub(crate) const SAPLING_TABLES_PREFIX: &str = "sapling"; /// A newtype wrapper for sqlite primary key values for the notes @@ -167,12 +172,9 @@ impl, P: consensus::Parameters> WalletRead for W wallet::block_fully_scanned(self.conn.borrow()) } - fn suggest_scan_ranges( - &self, - _batch_size: usize, - _limit: usize, - ) -> Result>, Self::Error> { - todo!() + fn suggest_scan_ranges(&self) -> Result, Self::Error> { + wallet::scanning::suggest_scan_ranges(self.conn.borrow(), None) + .map_err(SqliteClientError::from) } fn get_min_unspent_height(&self) -> Result, Self::Error> { @@ -401,16 +403,19 @@ impl WalletWrite for WalletDb blocks: Vec>, ) -> Result, Self::Error> { self.transactionally(|wdb| { - let start_position = blocks.first().map(|block| { - Position::from( - u64::from(block.metadata().sapling_tree_size()) - - u64::try_from(block.sapling_commitments().len()).unwrap(), + let start_positions = blocks.first().map(|block| { + ( + block.height(), + Position::from( + u64::from(block.metadata().sapling_tree_size()) + - u64::try_from(block.sapling_commitments().len()).unwrap(), + ), ) }); let mut wallet_note_ids = vec![]; let mut sapling_commitments = vec![]; let mut end_height = None; - + let mut note_positions = vec![]; for block in blocks.into_iter() { if end_height.iter().any(|prev| block.height() != *prev + 1) { return Err(SqliteClientError::NonSequentialBlocks); @@ -442,13 +447,21 @@ impl WalletWrite for WalletDb } } + note_positions.extend(block.transactions().iter().flat_map(|wtx| { + wtx.sapling_outputs + .iter() + .map(|out| out.note_commitment_tree_position()) + })); + end_height = Some(block.height()); sapling_commitments.extend(block.into_sapling_commitments().into_iter()); } // We will have a start position and an end height in all cases where `blocks` is // non-empty. - if let Some((start_position, end_height)) = start_position.zip(end_height) { + if let Some(((start_height, start_position), end_height)) = + start_positions.zip(end_height) + { // Update the Sapling note commitment tree with all newly read note commitments let mut sapling_commitments = sapling_commitments.into_iter(); wdb.with_sapling_tree_mut::<_, _, SqliteClientError>(move |sapling_tree| { @@ -458,12 +471,29 @@ impl WalletWrite for WalletDb // Update now-expired transactions that didn't get mined. wallet::update_expired_notes(wdb.conn.0, end_height)?; + + wallet::scanning::scan_complete( + wdb.conn.0, + &wdb.params, + Range { + start: start_height, + end: end_height + 1, + }, + ¬e_positions, + )?; } Ok(wallet_note_ids) }) } + fn update_chain_tip(&mut self, tip_height: BlockHeight) -> Result<(), Self::Error> { + let tx = self.conn.transaction()?; + wallet::scanning::update_chain_tip(&tx, &self.params, tip_height)?; + tx.commit()?; + Ok(()) + } + fn store_decrypted_tx( &mut self, d_tx: DecryptedTransaction, @@ -750,7 +780,7 @@ impl BlockSource for BlockDb { fn with_blocks( &self, from_height: Option, - limit: Option, + limit: Option, with_row: F, ) -> Result<(), data_api::chain::error::Error> where @@ -928,7 +958,7 @@ impl BlockSource for FsBlockDb { fn with_blocks( &self, from_height: Option, - limit: Option, + limit: Option, with_row: F, ) -> Result<(), data_api::chain::error::Error> where diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 6f90151bbd..4e66ed7cdf 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -68,6 +68,8 @@ use rusqlite::{self, named_params, OptionalExtension, ToSql}; use std::collections::HashMap; use std::convert::TryFrom; use std::io::{self, Cursor}; +use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; + use zcash_client_backend::data_api::ShieldedProtocol; use zcash_primitives::transaction::TransactionData; @@ -91,10 +93,13 @@ use zcash_client_backend::{ wallet::WalletTx, }; +use crate::VALIDATION_DEPTH; use crate::{ error::SqliteClientError, SqlTransaction, WalletCommitmentTrees, WalletDb, PRUNING_DEPTH, }; +use self::scanning::replace_queue_entries; + #[cfg(feature = "transparent-inputs")] use { crate::UtxoId, @@ -109,6 +114,7 @@ use { pub(crate) mod commitment_tree; pub mod init; pub(crate) mod sapling; +pub(crate) mod scanning; pub(crate) const BLOCK_SAPLING_FRONTIER_ABSENT: &[u8] = &[0x0]; @@ -629,8 +635,8 @@ pub(crate) fn block_metadata( ) -> Result, SqliteClientError> { conn.query_row( "SELECT height, hash, sapling_commitment_tree_size, sapling_tree - FROM blocks - WHERE height = :block_height", + FROM blocks + WHERE height = :block_height", named_params![":block_height": u32::from(block_height)], |row| { let height: u32 = row.get(0)?; @@ -800,7 +806,8 @@ pub(crate) fn truncate_to_height( // Un-mine transactions. conn.execute( - "UPDATE transactions SET block = NULL, tx_index = NULL WHERE block IS NOT NULL AND block > ?", + "UPDATE transactions SET block = NULL, tx_index = NULL + WHERE block IS NOT NULL AND block > ?", [u32::from(block_height)], )?; @@ -809,6 +816,27 @@ pub(crate) fn truncate_to_height( "DELETE FROM blocks WHERE height > ?", [u32::from(block_height)], )?; + + // Delete from the scanning queue any range with a start height greater than or equal to + // the truncation height, and truncate any remaining range by setting the end equal to + // the truncation height. + conn.execute( + "DELETE FROM scan_queue + WHERE block_range_start >= :block_height", + named_params![":block_height": u32::from(block_height)], + )?; + + conn.execute( + "UPDATE scan_queue + SET block_range_end = :block_height + WHERE block_range_end > :block_height", + named_params![":block_height": u32::from(block_height)], + )?; + + // Prioritize the range starting at the height we just rewound to for verification + let query_range = block_height..(block_height + VALIDATION_DEPTH); + let scan_range = ScanRange::from_parts(query_range.clone(), ScanPriority::Verify); + replace_queue_entries(conn, &query_range, Some(scan_range).into_iter())?; } Ok(()) diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 66efda12e2..5ac93a03f6 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -463,6 +463,16 @@ mod tests { FOREIGN KEY (block) REFERENCES blocks(height), CONSTRAINT witness_height UNIQUE (note, block) )", + "CREATE TABLE scan_queue ( + block_range_start INTEGER NOT NULL, + block_range_end INTEGER NOT NULL, + priority INTEGER NOT NULL, + CONSTRAINT range_start_uniq UNIQUE (block_range_start), + CONSTRAINT range_end_uniq UNIQUE (block_range_end), + CONSTRAINT range_bounds_order CHECK ( + block_range_start < block_range_end + ) + )", "CREATE TABLE schemer_migrations ( id blob PRIMARY KEY )", diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 8a238d00e4..7292fe6f08 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -11,7 +11,10 @@ use schemer_rusqlite::RusqliteMigration; use shardtree::ShardTree; use uuid::Uuid; -use zcash_client_backend::data_api::SAPLING_SHARD_HEIGHT; +use zcash_client_backend::data_api::{ + scanning::{ScanPriority, ScanRange}, + SAPLING_SHARD_HEIGHT, +}; use zcash_primitives::{ consensus::BlockHeight, merkle_tree::{read_commitment_tree, read_incremental_witness}, @@ -20,8 +23,10 @@ use zcash_primitives::{ use crate::{ wallet::{ + block_height_extrema, commitment_tree::SqliteShardStore, init::{migrations::received_notes_nullable_nf, WalletMigrationError}, + scanning::insert_queue_entries, }, PRUNING_DEPTH, SAPLING_TABLES_PREFIX, }; @@ -184,6 +189,27 @@ impl RusqliteMigration for Migration { } } + // Establish the scan queue & wallet history table + transaction.execute_batch( + "CREATE TABLE scan_queue ( + block_range_start INTEGER NOT NULL, + block_range_end INTEGER NOT NULL, + priority INTEGER NOT NULL, + CONSTRAINT range_start_uniq UNIQUE (block_range_start), + CONSTRAINT range_end_uniq UNIQUE (block_range_end), + CONSTRAINT range_bounds_order CHECK ( + block_range_start < block_range_end + ) + );", + )?; + + if let Some((start, end)) = block_height_extrema(transaction)? { + insert_queue_entries( + transaction, + Some(ScanRange::from_parts(start..end, ScanPriority::Historic)).iter(), + )?; + } + Ok(()) } diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs new file mode 100644 index 0000000000..edc9dfacce --- /dev/null +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -0,0 +1,905 @@ +use rusqlite::{self, named_params, types::Value, OptionalExtension}; +use std::cmp::{max, min, Ordering}; +use std::collections::BTreeSet; +use std::ops::Range; +use std::rc::Rc; +use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; + +use incrementalmerkletree::{Address, Position}; +use zcash_primitives::consensus::{self, BlockHeight, NetworkUpgrade}; + +use zcash_client_backend::data_api::SAPLING_SHARD_HEIGHT; + +use crate::error::SqliteClientError; +use crate::{PRUNING_DEPTH, VALIDATION_DEPTH}; + +use super::block_height_extrema; + +#[derive(Copy, Clone, PartialEq, Eq)] +enum Dominance { + Left, + Right, + Equal, +} + +pub(crate) fn parse_priority_code(code: i64) -> Option { + use ScanPriority::*; + match code { + 10 => Some(Scanned), + 20 => Some(Historic), + 30 => Some(OpenAdjacent), + 40 => Some(FoundNote), + 50 => Some(ChainTip), + 60 => Some(Verify), + _ => None, + } +} + +pub(crate) fn priority_code(priority: &ScanPriority) -> i64 { + use ScanPriority::*; + match priority { + Scanned => 10, + Historic => 20, + OpenAdjacent => 30, + FoundNote => 40, + ChainTip => 50, + Verify => 60, + } +} + +pub(crate) fn suggest_scan_ranges( + conn: &rusqlite::Connection, + min_priority: Option, +) -> Result, SqliteClientError> { + let mut stmt_scan_ranges = conn.prepare_cached( + "SELECT block_range_start, block_range_end, priority + FROM scan_queue + WHERE priority >= :min_priority + ORDER BY priority DESC, block_range_end DESC", + )?; + + let mut rows = stmt_scan_ranges.query(named_params![ + ":min_priority": priority_code(&min_priority.unwrap_or(ScanPriority::Historic)) + ])?; + + let mut result = vec![]; + while let Some(row) = rows.next()? { + let range = Range { + start: row.get::<_, u32>(0).map(BlockHeight::from)?, + end: row.get::<_, u32>(1).map(BlockHeight::from)?, + }; + let code = row.get::<_, i64>(2)?; + let priority = parse_priority_code(code).ok_or_else(|| { + SqliteClientError::CorruptedData(format!("scan priority not recognized: {}", code)) + })?; + + result.push(ScanRange::from_parts(range, priority)); + } + + Ok(result) +} + +// This implements the dominance rule for range priority. If the inserted range's priority is +// `Verify`, this replaces any existing priority. Otherwise, if the current priority is +// `Scanned`, this overwrites any priority +fn update_priority(current: ScanPriority, inserted: ScanPriority) -> ScanPriority { + match (current, inserted) { + (_, ScanPriority::Verify) => ScanPriority::Verify, + (ScanPriority::Scanned, _) => ScanPriority::Scanned, + (_, ScanPriority::Scanned) => ScanPriority::Scanned, + (a, b) => max(a, b), + } +} + +fn dominance(current: &ScanPriority, inserted: &ScanPriority) -> Dominance { + match (current, inserted) { + (_, ScanPriority::Verify) | (_, ScanPriority::Scanned) => Dominance::Right, + (ScanPriority::Scanned, _) => Dominance::Left, + (a, b) => match a.cmp(b) { + Ordering::Less => Dominance::Right, + Ordering::Equal => Dominance::Equal, + Ordering::Greater => Dominance::Left, + }, + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RangeOrdering { + LeftFirstDisjoint, + LeftFirstOverlap, + LeftContained, + Equal, + RightContained, + RightFirstOverlap, + RightFirstDisjoint, +} + +impl RangeOrdering { + fn cmp(a: &Range, b: &Range) -> Self { + use RangeOrdering::*; + assert!(a.start <= a.end && b.start <= b.end); + if a.end <= b.start { + LeftFirstDisjoint + } else if b.end <= a.start { + RightFirstDisjoint + } else if a.start < b.start { + if a.end >= b.end { + RightContained + } else { + LeftFirstOverlap + } + } else if b.start < a.start { + if b.end >= a.end { + LeftContained + } else { + RightFirstOverlap + } + } else { + // a.start == b.start + match a.end.cmp(&b.end) { + Ordering::Less => LeftContained, + Ordering::Equal => Equal, + Ordering::Greater => RightContained, + } + } + } +} + +enum Joined { + One(ScanRange), + Two(ScanRange, ScanRange), + Three(ScanRange, ScanRange, ScanRange), +} + +fn join_nonoverlapping(left: ScanRange, right: ScanRange) -> Joined { + assert!(left.block_range().end <= right.block_range().start); + + if left.block_range().end == right.block_range().start { + if left.priority() == right.priority() { + Joined::One(ScanRange::from_parts( + left.block_range().start..right.block_range().end, + left.priority(), + )) + } else { + Joined::Two(left, right) + } + } else { + // there is a gap that will need to be filled + let gap = ScanRange::from_parts( + left.block_range().end..right.block_range().start, + ScanPriority::Historic, + ); + + match join_nonoverlapping(left, gap) { + Joined::One(left) => join_nonoverlapping(left, right), + Joined::Two(left, gap) => match join_nonoverlapping(gap, right) { + Joined::One(right) => Joined::Two(left, right), + Joined::Two(gap, right) => Joined::Three(left, gap, right), + _ => unreachable!(), + }, + _ => unreachable!(), + } + } +} + +fn insert(current: ScanRange, to_insert: ScanRange) -> Joined { + enum Insert { + Left, + Right, + } + + fn join_overlapping(left: ScanRange, right: ScanRange, insert: Insert) -> Joined { + assert!( + left.block_range().start <= right.block_range().start + && left.block_range().end > right.block_range().start + ); + + // recompute the range dominance based upon the queue entry priorities + let dominance = match insert { + Insert::Left => dominance(&right.priority(), &left.priority()), + Insert::Right => dominance(&left.priority(), &right.priority()), + }; + + match dominance { + Dominance::Left => { + if let Some(right) = right.truncate_start(left.block_range().end) { + Joined::Two(left, right) + } else { + Joined::One(left) + } + } + Dominance::Equal => Joined::One(ScanRange::from_parts( + left.block_range().start..max(left.block_range().end, right.block_range().end), + left.priority(), + )), + Dominance::Right => { + if let Some(left) = left.truncate_end(right.block_range().start) { + if let Some(end) = left.truncate_start(right.block_range().end) { + Joined::Three(left, right, end) + } else { + Joined::Two(left, right) + } + } else if let Some(end) = left.truncate_start(right.block_range().end) { + Joined::Two(right, end) + } else { + Joined::One(right) + } + } + } + } + + use RangeOrdering::*; + match RangeOrdering::cmp(to_insert.block_range(), current.block_range()) { + LeftFirstDisjoint => join_nonoverlapping(to_insert, current), + LeftFirstOverlap | RightContained => join_overlapping(to_insert, current, Insert::Left), + Equal => Joined::One(ScanRange::from_parts( + to_insert.block_range().clone(), + update_priority(current.priority(), to_insert.priority()), + )), + RightFirstOverlap | LeftContained => join_overlapping(current, to_insert, Insert::Right), + RightFirstDisjoint => join_nonoverlapping(current, to_insert), + } +} + +#[derive(Debug, Clone)] +enum SpanningTree { + Leaf(ScanRange), + Parent { + span: Range, + left: Box, + right: Box, + }, +} + +impl SpanningTree { + fn span(&self) -> Range { + match self { + SpanningTree::Leaf(entry) => entry.block_range().clone(), + SpanningTree::Parent { span, .. } => span.clone(), + } + } + + fn from_joined(joined: Joined) -> Self { + match joined { + Joined::One(entry) => SpanningTree::Leaf(entry), + Joined::Two(left, right) => SpanningTree::Parent { + span: left.block_range().start..right.block_range().end, + left: Box::new(SpanningTree::Leaf(left)), + right: Box::new(SpanningTree::Leaf(right)), + }, + Joined::Three(left, mid, right) => SpanningTree::Parent { + span: left.block_range().start..right.block_range().end, + left: Box::new(SpanningTree::Leaf(left)), + right: Box::new(SpanningTree::Parent { + span: mid.block_range().start..right.block_range().end, + left: Box::new(SpanningTree::Leaf(mid)), + right: Box::new(SpanningTree::Leaf(right)), + }), + }, + } + } + + fn insert(self, to_insert: ScanRange) -> Self { + match self { + SpanningTree::Leaf(cur) => Self::from_joined(insert(cur, to_insert)), + SpanningTree::Parent { span, left, right } => { + // TODO: this algorithm always preserves the existing partition point, and does not + // do any rebalancing or unification of ranges within the tree; `into_vec` + // performes such unification and the tree being unbalanced should be fine given + // the relatively small number of ranges we should ordinarily be concerned with. + use RangeOrdering::*; + match RangeOrdering::cmp(&span, to_insert.block_range()) { + LeftFirstDisjoint => { + // extend the right-hand branch + SpanningTree::Parent { + span: left.span().start..to_insert.block_range().end, + left, + right: Box::new(right.insert(to_insert)), + } + } + LeftFirstOverlap => { + let split_point = left.span().end; + if split_point > to_insert.block_range().start { + let (l_insert, r_insert) = to_insert + .split_at(split_point) + .expect("Split point is within the range of to_insert"); + let left = Box::new(left.insert(l_insert)); + let right = Box::new(right.insert(r_insert)); + SpanningTree::Parent { + span: left.span().start..right.span().end, + left, + right, + } + } else { + // to_insert is fully contained in or equals the right child + SpanningTree::Parent { + span: left.span().start + ..max(right.span().end, to_insert.block_range().end), + left, + right: Box::new(right.insert(to_insert)), + } + } + } + RightContained => { + // to_insert is fully contained within the current span, so we will insert + // into one or both sides + let split_point = left.span().end; + if to_insert.block_range().start >= split_point { + // to_insert is fully contained in the right + SpanningTree::Parent { + span, + left, + right: Box::new(right.insert(to_insert)), + } + } else if to_insert.block_range().end <= split_point { + // to_insert is fully contained in the left + SpanningTree::Parent { + span, + left: Box::new(left.insert(to_insert)), + right, + } + } else { + // to_insert must be split. + let (l_insert, r_insert) = to_insert + .split_at(split_point) + .expect("Split point is within the range of to_insert"); + let left = Box::new(left.insert(l_insert)); + let right = Box::new(right.insert(r_insert)); + SpanningTree::Parent { + span: left.span().start..right.span().end, + left, + right, + } + } + } + Equal => { + if left.span().end > to_insert.block_range().start { + let (l_insert, r_insert) = to_insert + .split_at(left.span().end) + .expect("Split point is within the range of to_insert"); + let left = Box::new(left.insert(l_insert)); + let right = Box::new(right.insert(r_insert)); + SpanningTree::Parent { + span: left.span().start..right.span().end, + left, + right, + } + } else { + // to_insert is fully contained in the right subtree + right.insert(to_insert) + } + } + LeftContained => { + // the current span is fully contained within to_insert, so we will extend + // or overwrite both sides + let (l_insert, r_insert) = to_insert + .split_at(left.span().end) + .expect("Split point is within the range of to_insert"); + let left = Box::new(left.insert(l_insert)); + let right = Box::new(right.insert(r_insert)); + SpanningTree::Parent { + span: left.span().start..right.span().end, + left, + right, + } + } + RightFirstOverlap => { + let split_point = left.span().end; + if split_point < to_insert.block_range().end { + let (l_insert, r_insert) = to_insert + .split_at(split_point) + .expect("Split point is within the range of to_insert"); + let left = Box::new(left.insert(l_insert)); + let right = Box::new(right.insert(r_insert)); + SpanningTree::Parent { + span: left.span().start..right.span().end, + left, + right, + } + } else { + // to_insert is fully contained in or equals the left child + SpanningTree::Parent { + span: min(to_insert.block_range().start, left.span().start) + ..right.span().end, + left: Box::new(left.insert(to_insert)), + right, + } + } + } + RightFirstDisjoint => { + // extend the left-hand branch + SpanningTree::Parent { + span: to_insert.block_range().start..left.span().end, + left: Box::new(left.insert(to_insert)), + right, + } + } + } + } + } + } + + fn into_vec(self) -> Vec { + fn go(acc: &mut Vec, tree: SpanningTree) { + match tree { + SpanningTree::Leaf(entry) => { + if let Some(top) = acc.pop() { + match join_nonoverlapping(top, entry) { + Joined::One(entry) => acc.push(entry), + Joined::Two(l, r) => { + acc.push(l); + acc.push(r); + } + _ => unreachable!(), + } + } else { + acc.push(entry); + } + } + SpanningTree::Parent { left, right, .. } => { + go(acc, *left); + go(acc, *right); + } + } + } + + let mut acc = vec![]; + go(&mut acc, self); + acc + } +} + +pub(crate) fn insert_queue_entries<'a>( + conn: &rusqlite::Connection, + entries: impl Iterator, +) -> Result<(), rusqlite::Error> { + let mut stmt = conn.prepare_cached( + "INSERT INTO scan_queue (block_range_start, block_range_end, priority) + VALUES (:block_range_start, :block_range_end, :priority)", + )?; + + for entry in entries { + if entry.block_range().end > entry.block_range().start { + stmt.execute(named_params![ + ":block_range_start": u32::from(entry.block_range().start) , + ":block_range_end": u32::from(entry.block_range().end), + ":priority": priority_code(&entry.priority()) + ])?; + } + } + + Ok(()) +} + +pub(crate) fn replace_queue_entries( + conn: &rusqlite::Connection, + query_range: &Range, + mut entries: impl Iterator, +) -> Result<(), SqliteClientError> { + let (to_create, to_delete_ends) = { + let mut suggested_stmt = conn.prepare_cached( + "SELECT block_range_start, block_range_end, priority + FROM scan_queue + WHERE ( + -- the start is contained within the range + :start >= block_range_start + AND :start < block_range_end + ) + OR ( + -- the end is contained within the range + :end > block_range_start + AND :end <= block_range_end + ) + OR ( + -- start..end contains the entire range + block_range_start >= :start + AND block_range_end <= :end + ) + ORDER BY block_range_end", + )?; + + let mut rows = suggested_stmt.query(named_params![ + ":start": u32::from(query_range.start), + ":end": u32::from(query_range.end), + ])?; + + // Iterate over the ranges in the scan queue that overlaps the range that we have + // identified as needing to be fully scanned. For each such range add it to the + // spanning tree (these should all be nonoverlapping ranges, but we might coalesce + // some in the process). + let mut existing_ranges: Option = None; + let mut to_delete_ends: Vec = vec![]; + while let Some(row) = rows.next()? { + let entry = ScanRange::from_parts( + Range { + start: BlockHeight::from(row.get::<_, u32>(0)?), + end: BlockHeight::from(row.get::<_, u32>(1)?), + }, + { + let code = row.get::<_, i64>(2)?; + parse_priority_code(code).ok_or_else(|| { + SqliteClientError::CorruptedData(format!( + "scan priority not recognized: {}", + code + )) + })? + }, + ); + to_delete_ends.push(Value::from(u32::from(entry.block_range().end))); + existing_ranges = if let Some(cur) = existing_ranges { + Some(cur.insert(entry)) + } else { + Some(SpanningTree::Leaf(entry)) + }; + } + + // Update the tree that we read from the database, or if we didn't find any ranges + // start with the scanned range. + let mut to_create = match existing_ranges { + Some(cur) => entries.next().map(|entry| cur.insert(entry)), + None => entries.next().map(SpanningTree::Leaf), + }; + + for entry in entries { + to_create = to_create.map(|cur| cur.insert(entry)) + } + + (to_create, to_delete_ends) + }; + + if let Some(tree) = to_create { + let ends_ptr = Rc::new(to_delete_ends); + conn.execute( + "DELETE FROM scan_queue WHERE block_range_end IN rarray(:ends)", + named_params![":ends": ends_ptr], + )?; + + let scan_ranges = tree.into_vec(); + insert_queue_entries(conn, scan_ranges.iter())?; + } + + Ok(()) +} + +pub(crate) fn scan_complete( + conn: &rusqlite::Transaction<'_>, + params: &P, + range: Range, + wallet_note_positions: &[Position], +) -> Result<(), SqliteClientError> { + // Determine the range of block heights for which we will be updating the scan queue. + let extended_range = { + // If notes have been detected in the scan, we need to extend any adjacent un-scanned ranges to + // include the blocks needed to complete the note commitment tree subtrees containing the + // positions of the discovered notes. We will query by subtree index to find these bounds. + let required_subtrees = wallet_note_positions + .iter() + .map(|p| Address::above_position(SAPLING_SHARD_HEIGHT.into(), *p).index()) + .collect::>(); + + // we'll either have both min and max bounds, or we'll have neither + let subtree_bounds = required_subtrees + .iter() + .min() + .zip(required_subtrees.iter().max()); + + let mut sapling_shard_end_stmt = conn.prepare_cached( + "SELECT subtree_end_height + FROM sapling_tree_shards + WHERE shard_index = :shard_index", + )?; + + // if no notes belonging to the wallet were found, so don't need to extend the scanning + // range suggestions to include the associated subtrees, and our bounds are just the + // scanned range + subtree_bounds + .map(|(min_idx, max_idx)| { + let range_min = if *min_idx > 0 { + // get the block height of the end of the previous shard + sapling_shard_end_stmt + .query_row(named_params![":shard_index": *min_idx - 1], |row| { + row.get::<_, Option>(0) + .map(|opt| opt.map(BlockHeight::from)) + }) + .optional()? + .flatten() + } else { + // our lower bound is going to be the Sapling activation height + params.activation_height(NetworkUpgrade::Sapling) + }; + + // get the block height for the end of the current shard + let range_max = sapling_shard_end_stmt + .query_row(named_params![":shard_index": max_idx], |row| { + row.get::<_, Option>(0) + .map(|opt| opt.map(BlockHeight::from)) + }) + .optional()? + .flatten(); + + Ok::, rusqlite::Error>(match (range_min, range_max) { + (Some(start), Some(end)) => Range { start, end }, + (Some(start), None) => Range { + start, + end: range.end, + }, + (None, Some(end)) => Range { + start: range.start, + end, + }, + (None, None) => range.clone(), + }) + }) + .transpose() + .map_err(SqliteClientError::from) + }?; + + let query_range = extended_range.clone().unwrap_or_else(|| range.clone()); + + let scanned = ScanRange::from_parts(range.clone(), ScanPriority::Scanned); + let extensions = if let Some(extended) = extended_range { + vec![ + ScanRange::from_parts(extended.start..range.start, ScanPriority::FoundNote), + ScanRange::from_parts(range.end..extended.end, ScanPriority::FoundNote), + ] + } else { + vec![] + }; + + replace_queue_entries( + conn, + &query_range, + Some(scanned).into_iter().chain(extensions.into_iter()), + )?; + + Ok(()) +} + +pub(crate) fn update_chain_tip( + conn: &rusqlite::Transaction<'_>, + params: &P, + new_tip: BlockHeight, +) -> Result<(), SqliteClientError> { + // Read the maximum height from the shards table. + let shard_start_height = conn.query_row( + "SELECT MAX(subtree_end_height) + FROM sapling_tree_shards", + [], + |row| Ok(row.get::<_, Option>(0)?.map(BlockHeight::from)), + )?; + + // Create a scanning range for the fragment of the last shard leading up to new tip. + // However, only do so if the start of the shard is at a stable height. + let shard_entry = shard_start_height + .filter(|h| h < &new_tip) + .map(|h| ScanRange::from_parts(h..new_tip, ScanPriority::ChainTip)); + + // Create scanning ranges to either validate potentially invalid blocks at the wallet's view + // of the chain tip, + let tip_entry = block_height_extrema(conn)?.map(|(_, prior_tip)| { + // If we don't have shard metadata, this means we're doing linear scanning, so create a + // scan range from the prior tip to the current tip with `Historic` priority. + if shard_entry.is_none() { + ScanRange::from_parts(prior_tip..new_tip, ScanPriority::Historic) + } else { + // Determine the height to which we expect blocks retrieved from the block source to be stable + // and not subject to being reorg'ed. + let stable_height = new_tip.saturating_sub(PRUNING_DEPTH); + + // if the wallet's prior tip is above the stable height, prioritize the range between + // it and the new tip as `ChainTip`. Otherwise, prioritize the `VALIDATION_DEPTH` + // blocks above the wallet's prior tip as `Verify`. Since `scan_cached_blocks` + // retrieves the metadata for the block being connected to, the connectivity to the + // prior tip will always be checked. Since `Verify` ranges have maximum priority, even + // if the block source begins downloading blocks from the shard scan range (which ends + // at the stable height) the scanner should not attempt to scan those blocks until the + // tip range has been completely checked and any required rewinds have been performed. + if prior_tip >= stable_height { + // This may overlap the `shard_entry` range and if so will be coalesced with it. + ScanRange::from_parts(prior_tip..new_tip, ScanPriority::ChainTip) + } else { + // The prior tip is in the range that we now expect to be stable, so we need to verify + // and advance it up to at most the stable height. The shard entry will then cover + // the range to the new tip at the lower `ChainTip` priority. + ScanRange::from_parts( + prior_tip..max(stable_height, prior_tip + VALIDATION_DEPTH), + ScanPriority::Verify, + ) + } + } + }); + + let query_range = match (shard_entry.as_ref(), tip_entry.as_ref()) { + (Some(se), Some(te)) => Some(Range { + start: min(se.block_range().start, te.block_range().start), + end: max(se.block_range().end, te.block_range().end), + }), + (Some(se), None) => Some(se.block_range().clone()), + (None, Some(te)) => Some(te.block_range().clone()), + (None, None) => None, + }; + + if let Some(query_range) = query_range { + replace_queue_entries( + conn, + &query_range, + shard_entry.into_iter().chain(tip_entry.into_iter()), + )?; + } else { + // If we have neither shard data nor any existing block data in the database, we should also + // have no existing scan queue entries and can fall back to linear scanning from Sapling + // activation. + if let Some(sapling_activation) = params.activation_height(NetworkUpgrade::Sapling) { + let scan_range = + ScanRange::from_parts(sapling_activation..new_tip, ScanPriority::Historic); + insert_queue_entries(conn, Some(scan_range).iter())?; + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::ops::Range; + + use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; + use zcash_primitives::consensus::BlockHeight; + + use super::{RangeOrdering, SpanningTree}; + + #[test] + fn range_ordering() { + use super::RangeOrdering::*; + // Equal + assert_eq!(RangeOrdering::cmp(&(0..1), &(0..1)), Equal); + + // Disjoint or contiguous + assert_eq!(RangeOrdering::cmp(&(0..1), &(1..2)), LeftFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(1..2), &(0..1)), RightFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(0..1), &(2..3)), LeftFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(2..3), &(0..1)), RightFirstDisjoint); + + // Contained + assert_eq!(RangeOrdering::cmp(&(1..2), &(0..3)), LeftContained); + assert_eq!(RangeOrdering::cmp(&(0..3), &(1..2)), RightContained); + assert_eq!(RangeOrdering::cmp(&(0..1), &(0..3)), LeftContained); + assert_eq!(RangeOrdering::cmp(&(0..3), &(0..1)), RightContained); + assert_eq!(RangeOrdering::cmp(&(2..3), &(0..3)), LeftContained); + assert_eq!(RangeOrdering::cmp(&(0..3), &(2..3)), RightContained); + + // Overlap + assert_eq!(RangeOrdering::cmp(&(0..2), &(1..3)), LeftFirstOverlap); + assert_eq!(RangeOrdering::cmp(&(1..3), &(0..2)), RightFirstOverlap); + } + + fn scan_range(range: Range, priority: ScanPriority) -> ScanRange { + ScanRange::from_parts( + BlockHeight::from(range.start)..BlockHeight::from(range.end), + priority, + ) + } + + fn spanning_tree(to_insert: &[(Range, ScanPriority)]) -> Option { + to_insert.iter().fold(None, |acc, (range, priority)| { + let scan_range = scan_range(range.clone(), *priority); + match acc { + None => Some(SpanningTree::Leaf(scan_range)), + Some(t) => Some(t.insert(scan_range)), + } + }) + } + + #[test] + fn spanning_tree_insert_adjacent() { + use ScanPriority::*; + + let t = spanning_tree(&[ + (0..3, Historic), + (3..6, Scanned), + (6..8, ChainTip), + (8..10, ChainTip), + ]) + .unwrap(); + + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..3, Historic), + scan_range(3..6, Scanned), + scan_range(6..10, ChainTip), + ] + ); + } + + #[test] + fn spanning_tree_insert_overlaps() { + use ScanPriority::*; + + let t = spanning_tree(&[ + (0..3, Historic), + (2..5, Scanned), + (6..8, ChainTip), + (7..10, Scanned), + ]) + .unwrap(); + + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..2, Historic), + scan_range(2..5, Scanned), + scan_range(5..6, Historic), + scan_range(6..7, ChainTip), + scan_range(7..10, Scanned), + ] + ); + } + + #[test] + fn spanning_tree_dominance() { + use ScanPriority::*; + + let t = spanning_tree(&[(0..3, Verify), (2..8, Scanned), (6..10, Verify)]).unwrap(); + + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..2, Verify), + scan_range(2..6, Scanned), + scan_range(6..10, Verify), + ] + ); + + let t = spanning_tree(&[(0..3, Verify), (2..8, Historic), (6..10, Verify)]).unwrap(); + + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..3, Verify), + scan_range(3..6, Historic), + scan_range(6..10, Verify), + ] + ); + + let t = spanning_tree(&[(0..3, Scanned), (2..8, Verify), (6..10, Scanned)]).unwrap(); + + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..2, Scanned), + scan_range(2..6, Verify), + scan_range(6..10, Scanned), + ] + ); + + let t = spanning_tree(&[(0..3, Scanned), (2..8, Historic), (6..10, Scanned)]).unwrap(); + + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..3, Scanned), + scan_range(3..6, Historic), + scan_range(6..10, Scanned), + ] + ); + } + + #[test] + fn spanning_tree_insert_coalesce_scanned() { + use ScanPriority::*; + + let mut t = spanning_tree(&[ + (0..3, Historic), + (2..5, Scanned), + (6..8, ChainTip), + (7..10, Scanned), + ]) + .unwrap(); + + t = t.insert(scan_range(0..3, Scanned)); + t = t.insert(scan_range(5..8, Scanned)); + + assert_eq!(t.into_vec(), vec![scan_range(0..10, Scanned)]); + } +} diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index f2404ea94e..44f903ca0d 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -8,6 +8,7 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added +- `zcash_primitives::consensus::BlockHeight::saturating_sub` - `zcash_primitives::transaction::builder`: - `Builder::add_orchard_spend` - `Builder::add_orchard_output` diff --git a/zcash_primitives/src/consensus.rs b/zcash_primitives/src/consensus.rs index 563c698067..8c421c76ac 100644 --- a/zcash_primitives/src/consensus.rs +++ b/zcash_primitives/src/consensus.rs @@ -23,6 +23,12 @@ impl BlockHeight { pub const fn from_u32(v: u32) -> BlockHeight { BlockHeight(v) } + + /// Subtracts the provided value from this height, returning `H0` if this would result in + /// underflow of the wrapped `u32`. + pub fn saturating_sub(self, v: u32) -> BlockHeight { + BlockHeight(self.0.saturating_sub(v)) + } } impl fmt::Display for BlockHeight { From 6db4355fc46838b0c0bb91dd33ab7a67ad2c723e Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 7 Jul 2023 18:36:37 -0600 Subject: [PATCH 0057/1122] Fix an error in dominance calculation. --- zcash_client_sqlite/src/wallet/scanning.rs | 94 +++++++++++++++++----- 1 file changed, 72 insertions(+), 22 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index edc9dfacce..5313a330aa 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -1,7 +1,7 @@ use rusqlite::{self, named_params, types::Value, OptionalExtension}; use std::cmp::{max, min, Ordering}; use std::collections::BTreeSet; -use std::ops::Range; +use std::ops::{Not, Range}; use std::rc::Rc; use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; @@ -15,13 +15,39 @@ use crate::{PRUNING_DEPTH, VALIDATION_DEPTH}; use super::block_height_extrema; -#[derive(Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, Copy)] +enum Insert { + Left, + Right, +} + +impl Not for Insert { + type Output = Self; + + fn not(self) -> Self::Output { + match self { + Insert::Left => Insert::Right, + Insert::Right => Insert::Left, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Dominance { Left, Right, Equal, } +impl From for Dominance { + fn from(value: Insert) -> Self { + match value { + Insert::Left => Dominance::Left, + Insert::Right => Dominance::Right, + } + } +} + pub(crate) fn parse_priority_code(code: i64) -> Option { use ScanPriority::*; match code { @@ -91,14 +117,14 @@ fn update_priority(current: ScanPriority, inserted: ScanPriority) -> ScanPriorit } } -fn dominance(current: &ScanPriority, inserted: &ScanPriority) -> Dominance { +fn dominance(current: &ScanPriority, inserted: &ScanPriority, insert: Insert) -> Dominance { match (current, inserted) { - (_, ScanPriority::Verify) | (_, ScanPriority::Scanned) => Dominance::Right, - (ScanPriority::Scanned, _) => Dominance::Left, + (_, ScanPriority::Verify | ScanPriority::Scanned) => Dominance::from(insert), + (ScanPriority::Scanned, _) => Dominance::from(!insert), (a, b) => match a.cmp(b) { - Ordering::Less => Dominance::Right, + Ordering::Less => Dominance::from(insert), Ordering::Equal => Dominance::Equal, - Ordering::Greater => Dominance::Left, + Ordering::Greater => Dominance::from(!insert), }, } } @@ -183,11 +209,6 @@ fn join_nonoverlapping(left: ScanRange, right: ScanRange) -> Joined { } fn insert(current: ScanRange, to_insert: ScanRange) -> Joined { - enum Insert { - Left, - Right, - } - fn join_overlapping(left: ScanRange, right: ScanRange, insert: Insert) -> Joined { assert!( left.block_range().start <= right.block_range().start @@ -196,8 +217,8 @@ fn insert(current: ScanRange, to_insert: ScanRange) -> Joined { // recompute the range dominance based upon the queue entry priorities let dominance = match insert { - Insert::Left => dominance(&right.priority(), &left.priority()), - Insert::Right => dominance(&left.priority(), &right.priority()), + Insert::Left => dominance(&right.priority(), &left.priority(), insert), + Insert::Right => dominance(&left.priority(), &right.priority(), insert), }; match dominance { @@ -535,13 +556,15 @@ pub(crate) fn replace_queue_entries( // Update the tree that we read from the database, or if we didn't find any ranges // start with the scanned range. - let mut to_create = match existing_ranges { - Some(cur) => entries.next().map(|entry| cur.insert(entry)), - None => entries.next().map(SpanningTree::Leaf), + let mut to_create = match (existing_ranges, entries.next()) { + (Some(cur), Some(entry)) => Some(cur.insert(entry)), + (None, Some(entry)) => Some(SpanningTree::Leaf(entry)), + (Some(cur), None) => Some(cur), + (None, None) => None, }; for entry in entries { - to_create = to_create.map(|cur| cur.insert(entry)) + to_create = to_create.map(|cur| cur.insert(entry)); } (to_create, to_delete_ends) @@ -841,7 +864,6 @@ mod tests { use ScanPriority::*; let t = spanning_tree(&[(0..3, Verify), (2..8, Scanned), (6..10, Verify)]).unwrap(); - assert_eq!( t.into_vec(), vec![ @@ -852,7 +874,6 @@ mod tests { ); let t = spanning_tree(&[(0..3, Verify), (2..8, Historic), (6..10, Verify)]).unwrap(); - assert_eq!( t.into_vec(), vec![ @@ -863,7 +884,6 @@ mod tests { ); let t = spanning_tree(&[(0..3, Scanned), (2..8, Verify), (6..10, Scanned)]).unwrap(); - assert_eq!( t.into_vec(), vec![ @@ -874,7 +894,6 @@ mod tests { ); let t = spanning_tree(&[(0..3, Scanned), (2..8, Historic), (6..10, Scanned)]).unwrap(); - assert_eq!( t.into_vec(), vec![ @@ -883,6 +902,37 @@ mod tests { scan_range(6..10, Scanned), ] ); + + // a `ChainTip` insertion should not overwrite a scanned range. + let mut t = spanning_tree(&[(0..3, ChainTip), (3..5, Scanned), (5..7, ChainTip)]).unwrap(); + t = t.insert(scan_range(0..7, ChainTip)); + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..3, ChainTip), + scan_range(3..5, Scanned), + scan_range(5..7, ChainTip), + ] + ); + + let mut t = + spanning_tree(&[(280300..280310, FoundNote), (280310..280320, Scanned)]).unwrap(); + assert_eq!( + t.clone().into_vec(), + vec![ + scan_range(280300..280310, FoundNote), + scan_range(280310..280320, Scanned) + ] + ); + t = t.insert(scan_range(280300..280340, ChainTip)); + assert_eq!( + t.into_vec(), + vec![ + scan_range(280300..280310, ChainTip), + scan_range(280310..280320, Scanned), + scan_range(280320..280340, ChainTip) + ] + ); } #[test] From 352e1c709ac4c95074812dcabeba3cd5565371d2 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 7 Jul 2023 18:58:37 -0600 Subject: [PATCH 0058/1122] Add a test for `scan_complete` and `update_chain_tip` --- zcash_client_sqlite/src/wallet/scanning.rs | 158 ++++++++++++++++++++- 1 file changed, 155 insertions(+), 3 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 5313a330aa..fbb4f31dca 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -725,7 +725,7 @@ pub(crate) fn update_chain_tip( // and advance it up to at most the stable height. The shard entry will then cover // the range to the new tip at the lower `ChainTip` priority. ScanRange::from_parts( - prior_tip..max(stable_height, prior_tip + VALIDATION_DEPTH), + prior_tip..min(stable_height, prior_tip + VALIDATION_DEPTH), ScanPriority::Verify, ) } @@ -766,8 +766,28 @@ pub(crate) fn update_chain_tip( mod tests { use std::ops::Range; - use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; - use zcash_primitives::consensus::BlockHeight; + use incrementalmerkletree::{Hashable, Level}; + use rusqlite::Connection; + use secrecy::Secret; + use tempfile::NamedTempFile; + use zcash_client_backend::data_api::{ + chain::{scan_cached_blocks, CommitmentTreeRoot}, + scanning::{ScanPriority, ScanRange}, + WalletCommitmentTrees, WalletRead, WalletWrite, + }; + use zcash_primitives::{ + block::BlockHash, consensus::BlockHeight, sapling::Node, transaction::components::Amount, + }; + + use crate::{ + chain::init::init_cache_database, + tests::{ + self, fake_compact_block, init_test_accounts_table, insert_into_cache, + sapling_activation_height, AddressType, + }, + wallet::{init::init_wallet_db, scanning::suggest_scan_ranges}, + BlockDb, WalletDb, + }; use super::{RangeOrdering, SpanningTree}; @@ -952,4 +972,136 @@ mod tests { assert_eq!(t.into_vec(), vec![scan_range(0..10, Scanned)]); } + + #[test] + fn scan_complete() { + use ScanPriority::*; + + let cache_file = NamedTempFile::new().unwrap(); + let db_cache = BlockDb(Connection::open(cache_file.path()).unwrap()); + init_cache_database(&db_cache).unwrap(); + + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); + + // Add an account to the wallet + let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); + + assert_matches!( + // in the following, we don't care what the root hashes are, they just need to be + // distinct + db_data.put_sapling_subtree_roots( + 0, + &[ + CommitmentTreeRoot::from_parts( + sapling_activation_height() + 100, + Node::empty_root(Level::from(0)) + ), + CommitmentTreeRoot::from_parts( + sapling_activation_height() + 200, + Node::empty_root(Level::from(1)) + ), + CommitmentTreeRoot::from_parts( + sapling_activation_height() + 300, + Node::empty_root(Level::from(2)) + ), + ] + ), + Ok(()) + ); + + // We'll start inserting leaf notes 5 notes after the end of the third subtree, with a gap + // of 10 blocks. After `scan_cached_blocks`, the scan queue should have a requested scan + // range of 300..310 with `FoundNote` priority, 310..320 with `Scanned` priority. + let initial_sapling_tree_size = (0x1 << 16) * 3 + 5; + let initial_height = sapling_activation_height() + 310; + + let value = Amount::from_u64(50000).unwrap(); + let (mut cb, _) = fake_compact_block( + initial_height, + BlockHash([0; 32]), + &dfvk, + AddressType::DefaultExternal, + value, + initial_sapling_tree_size, + ); + insert_into_cache(&db_cache, &cb); + + for i in 1..=10 { + cb = fake_compact_block( + initial_height + i, + cb.hash(), + &dfvk, + AddressType::DefaultExternal, + Amount::from_u64(10000).unwrap(), + initial_sapling_tree_size + i, + ) + .0; + insert_into_cache(&db_cache, &cb); + } + + assert_matches!( + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + initial_height, + 10, + ), + Ok(()) + ); + + // Verify the that adjacent range needed to make the note spendable has been prioritized + let sap_active = u32::from(sapling_activation_height()); + assert_matches!( + db_data.suggest_scan_ranges(), + Ok(scan_ranges) if scan_ranges == vec![ + scan_range((sap_active + 300)..(sap_active + 310), FoundNote) + ] + ); + + // Check that the scanned range has been properly persisted + assert_matches!( + suggest_scan_ranges(&db_data.conn, Some(Scanned)), + Ok(scan_ranges) if scan_ranges == vec![ + scan_range((sap_active + 300)..(sap_active + 310), FoundNote), + scan_range((sap_active + 310)..(sap_active + 320), Scanned) + ] + ); + + // simulate the wallet going offline for a bit, update the chain tip to 30 blocks in the + // future + assert_matches!( + db_data.update_chain_tip(sapling_activation_height() + 340), + Ok(()) + ); + + // Check the scan range again, we should see a `ChainTip` range for the period we've been + // offline. + assert_matches!( + db_data.suggest_scan_ranges(), + Ok(scan_ranges) if scan_ranges == vec![ + scan_range((sap_active + 320)..(sap_active + 340), ChainTip), + scan_range((sap_active + 300)..(sap_active + 310), ChainTip) + ] + ); + + // Now simulate a jump ahead more than 100 blocks + assert_matches!( + db_data.update_chain_tip(sapling_activation_height() + 450), + Ok(()) + ); + + // Check the scan range again, we should see a `Validate` range for the previous wallet + // tip, and then a `ChainTip` for the remaining range. + assert_matches!( + db_data.suggest_scan_ranges(), + Ok(scan_ranges) if scan_ranges == vec![ + scan_range((sap_active + 319)..(sap_active + 329), Verify), + scan_range((sap_active + 329)..(sap_active + 450), ChainTip), + scan_range((sap_active + 300)..(sap_active + 310), ChainTip) + ] + ); + } } From e3aeb63e0aefbd609444a878b5d9463d6a45336a Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 12 Jul 2023 03:42:32 +0100 Subject: [PATCH 0059/1122] Fix off-by-one bugs with `ScanRange` end bounds Maximum chain heights are end-inclusive, while `ScanRange` is end-exclusive. Co-authored-by: Daira Hopwood --- zcash_client_sqlite/src/wallet.rs | 14 +++++++------- .../wallet/init/migrations/shardtree_support.rs | 8 +++++++- zcash_client_sqlite/src/wallet/scanning.rs | 17 ++++++++++------- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 4e66ed7cdf..b90a061072 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -817,20 +817,20 @@ pub(crate) fn truncate_to_height( [u32::from(block_height)], )?; - // Delete from the scanning queue any range with a start height greater than or equal to - // the truncation height, and truncate any remaining range by setting the end equal to - // the truncation height. + // Delete from the scanning queue any range with a start height greater than the + // truncation height, and then truncate any remaining range by setting the end + // equal to the truncation height + 1. conn.execute( "DELETE FROM scan_queue - WHERE block_range_start >= :block_height", + WHERE block_range_start > :block_height", named_params![":block_height": u32::from(block_height)], )?; conn.execute( "UPDATE scan_queue - SET block_range_end = :block_height - WHERE block_range_end > :block_height", - named_params![":block_height": u32::from(block_height)], + SET block_range_end = :end_height + WHERE block_range_end > :end_height", + named_params![":end_height": u32::from(block_height + 1)], )?; // Prioritize the range starting at the height we just rewound to for verification diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 7292fe6f08..a00a5b97bb 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -204,9 +204,15 @@ impl RusqliteMigration for Migration { )?; if let Some((start, end)) = block_height_extrema(transaction)? { + // `ScanRange` uses an exclusive upper bound. + let chain_end = end + 1; insert_queue_entries( transaction, - Some(ScanRange::from_parts(start..end, ScanPriority::Historic)).iter(), + Some(ScanRange::from_parts( + start..chain_end, + ScanPriority::Historic, + )) + .iter(), )?; } diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index fbb4f31dca..03bacf582c 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -683,6 +683,9 @@ pub(crate) fn update_chain_tip( params: &P, new_tip: BlockHeight, ) -> Result<(), SqliteClientError> { + // `ScanRange` uses an exclusive upper bound. + let chain_end = new_tip + 1; + // Read the maximum height from the shards table. let shard_start_height = conn.query_row( "SELECT MAX(subtree_end_height) @@ -694,8 +697,8 @@ pub(crate) fn update_chain_tip( // Create a scanning range for the fragment of the last shard leading up to new tip. // However, only do so if the start of the shard is at a stable height. let shard_entry = shard_start_height - .filter(|h| h < &new_tip) - .map(|h| ScanRange::from_parts(h..new_tip, ScanPriority::ChainTip)); + .filter(|h| h < &chain_end) + .map(|h| ScanRange::from_parts(h..chain_end, ScanPriority::ChainTip)); // Create scanning ranges to either validate potentially invalid blocks at the wallet's view // of the chain tip, @@ -703,7 +706,7 @@ pub(crate) fn update_chain_tip( // If we don't have shard metadata, this means we're doing linear scanning, so create a // scan range from the prior tip to the current tip with `Historic` priority. if shard_entry.is_none() { - ScanRange::from_parts(prior_tip..new_tip, ScanPriority::Historic) + ScanRange::from_parts(prior_tip..chain_end, ScanPriority::Historic) } else { // Determine the height to which we expect blocks retrieved from the block source to be stable // and not subject to being reorg'ed. @@ -719,7 +722,7 @@ pub(crate) fn update_chain_tip( // tip range has been completely checked and any required rewinds have been performed. if prior_tip >= stable_height { // This may overlap the `shard_entry` range and if so will be coalesced with it. - ScanRange::from_parts(prior_tip..new_tip, ScanPriority::ChainTip) + ScanRange::from_parts(prior_tip..chain_end, ScanPriority::ChainTip) } else { // The prior tip is in the range that we now expect to be stable, so we need to verify // and advance it up to at most the stable height. The shard entry will then cover @@ -754,7 +757,7 @@ pub(crate) fn update_chain_tip( // activation. if let Some(sapling_activation) = params.activation_height(NetworkUpgrade::Sapling) { let scan_range = - ScanRange::from_parts(sapling_activation..new_tip, ScanPriority::Historic); + ScanRange::from_parts(sapling_activation..chain_end, ScanPriority::Historic); insert_queue_entries(conn, Some(scan_range).iter())?; } } @@ -1082,7 +1085,7 @@ mod tests { assert_matches!( db_data.suggest_scan_ranges(), Ok(scan_ranges) if scan_ranges == vec![ - scan_range((sap_active + 320)..(sap_active + 340), ChainTip), + scan_range((sap_active + 320)..(sap_active + 341), ChainTip), scan_range((sap_active + 300)..(sap_active + 310), ChainTip) ] ); @@ -1099,7 +1102,7 @@ mod tests { db_data.suggest_scan_ranges(), Ok(scan_ranges) if scan_ranges == vec![ scan_range((sap_active + 319)..(sap_active + 329), Verify), - scan_range((sap_active + 329)..(sap_active + 450), ChainTip), + scan_range((sap_active + 329)..(sap_active + 451), ChainTip), scan_range((sap_active + 300)..(sap_active + 310), ChainTip) ] ); From ed4b6dc9b963fa1e7da68b8e672b74af2b5e1174 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 12 Jul 2023 03:47:51 +0100 Subject: [PATCH 0060/1122] Fix bugs in `ScanRange::truncate_*` --- zcash_client_backend/src/data_api/scanning.rs | 78 +++++++++++++++++-- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/zcash_client_backend/src/data_api/scanning.rs b/zcash_client_backend/src/data_api/scanning.rs index 6a48297807..2c51daa01e 100644 --- a/zcash_client_backend/src/data_api/scanning.rs +++ b/zcash_client_backend/src/data_api/scanning.rs @@ -59,13 +59,13 @@ impl ScanRange { /// Shifts the start of the block range to the right if `block_height > /// self.block_range().start`. Returns `None` if the resulting range would - /// be empty. + /// be empty (or the range was already empty). pub fn truncate_start(&self, block_height: BlockHeight) -> Option { - if block_height >= self.block_range.end { + if block_height >= self.block_range.end || self.is_empty() { None } else { Some(ScanRange { - block_range: block_height..self.block_range.end, + block_range: self.block_range.start.max(block_height)..self.block_range.end, priority: self.priority, }) } @@ -73,13 +73,13 @@ impl ScanRange { /// Shifts the end of the block range to the left if `block_height < /// self.block_range().end`. Returns `None` if the resulting range would - /// be empty. + /// be empty (or the range was already empty). pub fn truncate_end(&self, block_height: BlockHeight) -> Option { - if block_height <= self.block_range.start { + if block_height <= self.block_range.start || self.is_empty() { None } else { Some(ScanRange { - block_range: self.block_range.start..block_height, + block_range: self.block_range.start..self.block_range.end.min(block_height), priority: self.priority, }) } @@ -105,3 +105,69 @@ impl ScanRange { } } } + +#[cfg(test)] +mod tests { + use super::{ScanPriority, ScanRange}; + + fn scan_range(start: u32, end: u32) -> ScanRange { + ScanRange::from_parts((start.into())..(end.into()), ScanPriority::Scanned) + } + + #[test] + fn truncate_start() { + let r = scan_range(5, 8); + + assert_eq!(r.truncate_start(4.into()), Some(scan_range(5, 8))); + assert_eq!(r.truncate_start(5.into()), Some(scan_range(5, 8))); + assert_eq!(r.truncate_start(6.into()), Some(scan_range(6, 8))); + assert_eq!(r.truncate_start(7.into()), Some(scan_range(7, 8))); + assert_eq!(r.truncate_start(8.into()), None); + assert_eq!(r.truncate_start(9.into()), None); + + let empty = scan_range(5, 5); + assert_eq!(empty.truncate_start(4.into()), None); + assert_eq!(empty.truncate_start(5.into()), None); + assert_eq!(empty.truncate_start(6.into()), None); + } + + #[test] + fn truncate_end() { + let r = scan_range(5, 8); + + assert_eq!(r.truncate_end(9.into()), Some(scan_range(5, 8))); + assert_eq!(r.truncate_end(8.into()), Some(scan_range(5, 8))); + assert_eq!(r.truncate_end(7.into()), Some(scan_range(5, 7))); + assert_eq!(r.truncate_end(6.into()), Some(scan_range(5, 6))); + assert_eq!(r.truncate_end(5.into()), None); + assert_eq!(r.truncate_end(4.into()), None); + + let empty = scan_range(5, 5); + assert_eq!(empty.truncate_end(4.into()), None); + assert_eq!(empty.truncate_end(5.into()), None); + assert_eq!(empty.truncate_end(6.into()), None); + } + + #[test] + fn split_at() { + let r = scan_range(5, 8); + + assert_eq!(r.split_at(4.into()), None); + assert_eq!(r.split_at(5.into()), None); + assert_eq!( + r.split_at(6.into()), + Some((scan_range(5, 6), scan_range(6, 8))) + ); + assert_eq!( + r.split_at(7.into()), + Some((scan_range(5, 7), scan_range(7, 8))) + ); + assert_eq!(r.split_at(8.into()), None); + assert_eq!(r.split_at(9.into()), None); + + let empty = scan_range(5, 5); + assert_eq!(empty.split_at(4.into()), None); + assert_eq!(empty.split_at(5.into()), None); + assert_eq!(empty.split_at(6.into()), None); + } +} From 2c0acac1bd33a53e07eaedd668b98a3f7e4dcab3 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 12 Jul 2023 03:56:37 +0100 Subject: [PATCH 0061/1122] Use explicit arguments for internal `suggest_scan_ranges` helper Co-authored-by: Daira Hopwood --- zcash_client_sqlite/src/lib.rs | 4 ++-- zcash_client_sqlite/src/wallet/scanning.rs | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 57aa166c2b..fe1f3b5ac3 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -57,7 +57,7 @@ use zcash_client_backend::{ data_api::{ self, chain::{BlockSource, CommitmentTreeRoot}, - scanning::ScanRange, + scanning::{ScanPriority, ScanRange}, BlockMetadata, DecryptedTransaction, NullifierQuery, PoolType, Recipient, ScannedBlock, SentTransaction, ShieldedProtocol, WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, @@ -173,7 +173,7 @@ impl, P: consensus::Parameters> WalletRead for W } fn suggest_scan_ranges(&self) -> Result, Self::Error> { - wallet::scanning::suggest_scan_ranges(self.conn.borrow(), None) + wallet::scanning::suggest_scan_ranges(self.conn.borrow(), ScanPriority::Historic) .map_err(SqliteClientError::from) } diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 03bacf582c..a34d18703c 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -75,7 +75,7 @@ pub(crate) fn priority_code(priority: &ScanPriority) -> i64 { pub(crate) fn suggest_scan_ranges( conn: &rusqlite::Connection, - min_priority: Option, + min_priority: ScanPriority, ) -> Result, SqliteClientError> { let mut stmt_scan_ranges = conn.prepare_cached( "SELECT block_range_start, block_range_end, priority @@ -84,9 +84,8 @@ pub(crate) fn suggest_scan_ranges( ORDER BY priority DESC, block_range_end DESC", )?; - let mut rows = stmt_scan_ranges.query(named_params![ - ":min_priority": priority_code(&min_priority.unwrap_or(ScanPriority::Historic)) - ])?; + let mut rows = + stmt_scan_ranges.query(named_params![":min_priority": priority_code(&min_priority)])?; let mut result = vec![]; while let Some(row) = rows.next()? { @@ -1066,7 +1065,7 @@ mod tests { // Check that the scanned range has been properly persisted assert_matches!( - suggest_scan_ranges(&db_data.conn, Some(Scanned)), + suggest_scan_ranges(&db_data.conn, Scanned), Ok(scan_ranges) if scan_ranges == vec![ scan_range((sap_active + 300)..(sap_active + 310), FoundNote), scan_range((sap_active + 310)..(sap_active + 320), Scanned) From bb920341a6bcca7d67f1e1ff5803c9faed8cba73 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 12 Jul 2023 17:04:31 +0100 Subject: [PATCH 0062/1122] Fix bug in `RightFirstDisjoint` insertion logic --- zcash_client_sqlite/src/wallet/scanning.rs | 36 +++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index a34d18703c..d5df3d6c76 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -429,7 +429,7 @@ impl SpanningTree { RightFirstDisjoint => { // extend the left-hand branch SpanningTree::Parent { - span: to_insert.block_range().start..left.span().end, + span: to_insert.block_range().start..right.span().end, left: Box::new(left.insert(to_insert)), right, } @@ -881,6 +881,40 @@ mod tests { ); } + #[test] + fn spanning_tree_insert_rfd_span() { + use ScanPriority::*; + + // This sequence of insertions causes a RightFirstDisjoint on the last insertion, + // which originally had a bug that caused the parent's span to only cover its left + // child. The bug was otherwise unobservable as the insertion logic was able to + // heal this specific kind of bug. + let t = spanning_tree(&[ + // 6..8 + (6..8, Scanned), + // 6..12 + // 6..8 8..12 + // 8..10 10..12 + (10..12, ChainTip), + // 3..12 + // 3..8 8..12 + // 3..6 6..8 8..10 10..12 + (3..6, Historic), + ]) + .unwrap(); + + assert_eq!(t.span(), (3.into())..(12.into())); + assert_eq!( + t.into_vec(), + vec![ + scan_range(3..6, Historic), + scan_range(6..8, Scanned), + scan_range(8..10, Historic), + scan_range(10..12, ChainTip), + ] + ); + } + #[test] fn spanning_tree_dominance() { use ScanPriority::*; From f7163e9dd9093cb3db870fbb0311ad73e7af9015 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 12 Jul 2023 17:12:14 +0100 Subject: [PATCH 0063/1122] Code simplifications Co-authored-by: Daira Hopwood --- zcash_client_backend/src/data_api/scanning.rs | 26 +- zcash_client_sqlite/src/lib.rs | 21 +- zcash_client_sqlite/src/wallet/scanning.rs | 283 +++++++----------- 3 files changed, 127 insertions(+), 203 deletions(-) diff --git a/zcash_client_backend/src/data_api/scanning.rs b/zcash_client_backend/src/data_api/scanning.rs index 2c51daa01e..c0315f2408 100644 --- a/zcash_client_backend/src/data_api/scanning.rs +++ b/zcash_client_backend/src/data_api/scanning.rs @@ -48,7 +48,7 @@ impl ScanRange { /// Returns whether or not the scan range is empty. pub fn is_empty(&self) -> bool { - self.block_range.end == self.block_range.start + self.block_range.is_empty() } /// Returns the number of blocks in the scan range. @@ -89,20 +89,16 @@ impl ScanRange { /// end of the first range returned and the start of the second. Returns `None` if /// `p <= self.block_range().start || p >= self.block_range().end`. pub fn split_at(&self, p: BlockHeight) -> Option<(Self, Self)> { - if p > self.block_range.start && p < self.block_range.end { - Some(( - ScanRange { - block_range: self.block_range.start..p, - priority: self.priority, - }, - ScanRange { - block_range: p..self.block_range.end, - priority: self.priority, - }, - )) - } else { - None - } + (p > self.block_range.start && p < self.block_range.end).then_some(( + ScanRange { + block_range: self.block_range.start..p, + priority: self.priority, + }, + ScanRange { + block_range: p..self.block_range.end, + priority: self.priority, + }, + )) } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index fe1f3b5ac3..94fb88f57a 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -414,10 +414,13 @@ impl WalletWrite for WalletDb }); let mut wallet_note_ids = vec![]; let mut sapling_commitments = vec![]; - let mut end_height = None; + let mut last_scanned_height = None; let mut note_positions = vec![]; for block in blocks.into_iter() { - if end_height.iter().any(|prev| block.height() != *prev + 1) { + if last_scanned_height + .iter() + .any(|prev| block.height() != *prev + 1) + { return Err(SqliteClientError::NonSequentialBlocks); } @@ -453,14 +456,14 @@ impl WalletWrite for WalletDb .map(|out| out.note_commitment_tree_position()) })); - end_height = Some(block.height()); + last_scanned_height = Some(block.height()); sapling_commitments.extend(block.into_sapling_commitments().into_iter()); } - // We will have a start position and an end height in all cases where `blocks` is - // non-empty. - if let Some(((start_height, start_position), end_height)) = - start_positions.zip(end_height) + // We will have a start position and a last scanned height in all cases where + // `blocks` is non-empty. + if let Some(((start_height, start_position), last_scanned_height)) = + start_positions.zip(last_scanned_height) { // Update the Sapling note commitment tree with all newly read note commitments let mut sapling_commitments = sapling_commitments.into_iter(); @@ -470,14 +473,14 @@ impl WalletWrite for WalletDb })?; // Update now-expired transactions that didn't get mined. - wallet::update_expired_notes(wdb.conn.0, end_height)?; + wallet::update_expired_notes(wdb.conn.0, last_scanned_height)?; wallet::scanning::scan_complete( wdb.conn.0, &wdb.params, Range { start: start_height, - end: end_height + 1, + end: last_scanned_height + 1, }, ¬e_positions, )?; diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index d5df3d6c76..e334781dce 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -107,24 +107,13 @@ pub(crate) fn suggest_scan_ranges( // This implements the dominance rule for range priority. If the inserted range's priority is // `Verify`, this replaces any existing priority. Otherwise, if the current priority is // `Scanned`, this overwrites any priority -fn update_priority(current: ScanPriority, inserted: ScanPriority) -> ScanPriority { - match (current, inserted) { - (_, ScanPriority::Verify) => ScanPriority::Verify, - (ScanPriority::Scanned, _) => ScanPriority::Scanned, - (_, ScanPriority::Scanned) => ScanPriority::Scanned, - (a, b) => max(a, b), - } -} - fn dominance(current: &ScanPriority, inserted: &ScanPriority, insert: Insert) -> Dominance { - match (current, inserted) { - (_, ScanPriority::Verify | ScanPriority::Scanned) => Dominance::from(insert), - (ScanPriority::Scanned, _) => Dominance::from(!insert), - (a, b) => match a.cmp(b) { - Ordering::Less => Dominance::from(insert), - Ordering::Equal => Dominance::Equal, - Ordering::Greater => Dominance::from(!insert), - }, + match (current.cmp(inserted), (current, inserted)) { + (Ordering::Equal, _) => Dominance::Equal, + (_, (_, ScanPriority::Verify | ScanPriority::Scanned)) => Dominance::from(insert), + (_, (ScanPriority::Scanned, _)) => Dominance::from(!insert), + (Ordering::Less, _) => Dominance::from(insert), + (Ordering::Greater, _) => Dominance::from(!insert), } } @@ -141,31 +130,16 @@ enum RangeOrdering { impl RangeOrdering { fn cmp(a: &Range, b: &Range) -> Self { - use RangeOrdering::*; + use Ordering::*; assert!(a.start <= a.end && b.start <= b.end); - if a.end <= b.start { - LeftFirstDisjoint - } else if b.end <= a.start { - RightFirstDisjoint - } else if a.start < b.start { - if a.end >= b.end { - RightContained - } else { - LeftFirstOverlap - } - } else if b.start < a.start { - if b.end >= a.end { - LeftContained - } else { - RightFirstOverlap - } - } else { - // a.start == b.start - match a.end.cmp(&b.end) { - Ordering::Less => LeftContained, - Ordering::Equal => Equal, - Ordering::Greater => RightContained, - } + match (a.start.cmp(&b.start), a.end.cmp(&b.end)) { + _ if a.end <= b.start => RangeOrdering::LeftFirstDisjoint, + _ if b.end <= a.start => RangeOrdering::RightFirstDisjoint, + (Less, Less) => RangeOrdering::LeftFirstOverlap, + (Equal, Less) | (Greater, Less) | (Greater, Equal) => RangeOrdering::LeftContained, + (Equal, Equal) => RangeOrdering::Equal, + (Equal, Greater) | (Less, Greater) | (Less, Equal) => RangeOrdering::RightContained, + (Greater, Greater) => RangeOrdering::RightFirstOverlap, } } } @@ -196,9 +170,9 @@ fn join_nonoverlapping(left: ScanRange, right: ScanRange) -> Joined { ); match join_nonoverlapping(left, gap) { - Joined::One(left) => join_nonoverlapping(left, right), + Joined::One(merged) => join_nonoverlapping(merged, right), Joined::Two(left, gap) => match join_nonoverlapping(gap, right) { - Joined::One(right) => Joined::Two(left, right), + Joined::One(merged) => Joined::Two(left, merged), Joined::Two(gap, right) => Joined::Three(left, gap, right), _ => unreachable!(), }, @@ -232,19 +206,15 @@ fn insert(current: ScanRange, to_insert: ScanRange) -> Joined { left.block_range().start..max(left.block_range().end, right.block_range().end), left.priority(), )), - Dominance::Right => { - if let Some(left) = left.truncate_end(right.block_range().start) { - if let Some(end) = left.truncate_start(right.block_range().end) { - Joined::Three(left, right, end) - } else { - Joined::Two(left, right) - } - } else if let Some(end) = left.truncate_start(right.block_range().end) { - Joined::Two(right, end) - } else { - Joined::One(right) - } - } + Dominance::Right => match ( + left.truncate_end(right.block_range().start), + left.truncate_start(right.block_range().end), + ) { + (Some(before), Some(after)) => Joined::Three(before, right, after), + (Some(before), None) => Joined::Two(before, right), + (None, Some(after)) => Joined::Two(right, after), + (None, None) => Joined::One(right), + }, } } @@ -254,7 +224,10 @@ fn insert(current: ScanRange, to_insert: ScanRange) -> Joined { LeftFirstOverlap | RightContained => join_overlapping(to_insert, current, Insert::Left), Equal => Joined::One(ScanRange::from_parts( to_insert.block_range().clone(), - update_priority(current.priority(), to_insert.priority()), + match dominance(¤t.priority(), &to_insert.priority(), Insert::Right) { + Dominance::Left | Dominance::Equal => current.priority(), + Dominance::Right => to_insert.priority(), + }, )), RightFirstOverlap | LeftContained => join_overlapping(current, to_insert, Insert::Right), RightFirstDisjoint => join_nonoverlapping(current, to_insert), @@ -299,6 +272,36 @@ impl SpanningTree { } } + fn from_insert( + left: Box, + right: Box, + to_insert: ScanRange, + insert: Insert, + ) -> Self { + let (left, right) = match insert { + Insert::Left => (Box::new(left.insert(to_insert)), right), + Insert::Right => (left, Box::new(right.insert(to_insert))), + }; + SpanningTree::Parent { + span: left.span().start..right.span().end, + left, + right, + } + } + + fn from_split(left: Self, right: Self, to_insert: ScanRange, split_point: BlockHeight) -> Self { + let (l_insert, r_insert) = to_insert + .split_at(split_point) + .expect("Split point is within the range of to_insert"); + let left = Box::new(left.insert(l_insert)); + let right = Box::new(right.insert(r_insert)); + SpanningTree::Parent { + span: left.span().start..right.span().end, + left, + right, + } + } + fn insert(self, to_insert: ScanRange) -> Self { match self { SpanningTree::Leaf(cur) => Self::from_joined(insert(cur, to_insert)), @@ -311,33 +314,15 @@ impl SpanningTree { match RangeOrdering::cmp(&span, to_insert.block_range()) { LeftFirstDisjoint => { // extend the right-hand branch - SpanningTree::Parent { - span: left.span().start..to_insert.block_range().end, - left, - right: Box::new(right.insert(to_insert)), - } + Self::from_insert(left, right, to_insert, Insert::Right) } LeftFirstOverlap => { let split_point = left.span().end; if split_point > to_insert.block_range().start { - let (l_insert, r_insert) = to_insert - .split_at(split_point) - .expect("Split point is within the range of to_insert"); - let left = Box::new(left.insert(l_insert)); - let right = Box::new(right.insert(r_insert)); - SpanningTree::Parent { - span: left.span().start..right.span().end, - left, - right, - } + Self::from_split(*left, *right, to_insert, split_point) } else { // to_insert is fully contained in or equals the right child - SpanningTree::Parent { - span: left.span().start - ..max(right.span().end, to_insert.block_range().end), - left, - right: Box::new(right.insert(to_insert)), - } + Self::from_insert(left, right, to_insert, Insert::Right) } } RightContained => { @@ -346,44 +331,19 @@ impl SpanningTree { let split_point = left.span().end; if to_insert.block_range().start >= split_point { // to_insert is fully contained in the right - SpanningTree::Parent { - span, - left, - right: Box::new(right.insert(to_insert)), - } + Self::from_insert(left, right, to_insert, Insert::Right) } else if to_insert.block_range().end <= split_point { // to_insert is fully contained in the left - SpanningTree::Parent { - span, - left: Box::new(left.insert(to_insert)), - right, - } + Self::from_insert(left, right, to_insert, Insert::Left) } else { // to_insert must be split. - let (l_insert, r_insert) = to_insert - .split_at(split_point) - .expect("Split point is within the range of to_insert"); - let left = Box::new(left.insert(l_insert)); - let right = Box::new(right.insert(r_insert)); - SpanningTree::Parent { - span: left.span().start..right.span().end, - left, - right, - } + Self::from_split(*left, *right, to_insert, split_point) } } Equal => { - if left.span().end > to_insert.block_range().start { - let (l_insert, r_insert) = to_insert - .split_at(left.span().end) - .expect("Split point is within the range of to_insert"); - let left = Box::new(left.insert(l_insert)); - let right = Box::new(right.insert(r_insert)); - SpanningTree::Parent { - span: left.span().start..right.span().end, - left, - right, - } + let split_point = left.span().end; + if split_point > to_insert.block_range().start { + Self::from_split(*left, *right, to_insert, split_point) } else { // to_insert is fully contained in the right subtree right.insert(to_insert) @@ -392,47 +352,21 @@ impl SpanningTree { LeftContained => { // the current span is fully contained within to_insert, so we will extend // or overwrite both sides - let (l_insert, r_insert) = to_insert - .split_at(left.span().end) - .expect("Split point is within the range of to_insert"); - let left = Box::new(left.insert(l_insert)); - let right = Box::new(right.insert(r_insert)); - SpanningTree::Parent { - span: left.span().start..right.span().end, - left, - right, - } + let split_point = left.span().end; + Self::from_split(*left, *right, to_insert, split_point) } RightFirstOverlap => { let split_point = left.span().end; if split_point < to_insert.block_range().end { - let (l_insert, r_insert) = to_insert - .split_at(split_point) - .expect("Split point is within the range of to_insert"); - let left = Box::new(left.insert(l_insert)); - let right = Box::new(right.insert(r_insert)); - SpanningTree::Parent { - span: left.span().start..right.span().end, - left, - right, - } + Self::from_split(*left, *right, to_insert, split_point) } else { // to_insert is fully contained in or equals the left child - SpanningTree::Parent { - span: min(to_insert.block_range().start, left.span().start) - ..right.span().end, - left: Box::new(left.insert(to_insert)), - right, - } + Self::from_insert(left, right, to_insert, Insert::Left) } } RightFirstDisjoint => { // extend the left-hand branch - SpanningTree::Parent { - span: to_insert.block_range().start..right.span().end, - left: Box::new(left.insert(to_insert)), - right, - } + Self::from_insert(left, right, to_insert, Insert::Left) } } } @@ -445,7 +379,7 @@ impl SpanningTree { SpanningTree::Leaf(entry) => { if let Some(top) = acc.pop() { match join_nonoverlapping(top, entry) { - Joined::One(entry) => acc.push(entry), + Joined::One(merged) => acc.push(merged), Joined::Two(l, r) => { acc.push(l); acc.push(r); @@ -479,7 +413,7 @@ pub(crate) fn insert_queue_entries<'a>( )?; for entry in entries { - if entry.block_range().end > entry.block_range().start { + if !entry.is_empty() { stmt.execute(named_params![ ":block_range_start": u32::from(entry.block_range().start) , ":block_range_end": u32::from(entry.block_range().end), @@ -494,7 +428,7 @@ pub(crate) fn insert_queue_entries<'a>( pub(crate) fn replace_queue_entries( conn: &rusqlite::Connection, query_range: &Range, - mut entries: impl Iterator, + entries: impl Iterator, ) -> Result<(), SqliteClientError> { let (to_create, to_delete_ends) = { let mut suggested_stmt = conn.prepare_cached( @@ -527,7 +461,7 @@ pub(crate) fn replace_queue_entries( // identified as needing to be fully scanned. For each such range add it to the // spanning tree (these should all be nonoverlapping ranges, but we might coalesce // some in the process). - let mut existing_ranges: Option = None; + let mut to_create: Option = None; let mut to_delete_ends: Vec = vec![]; while let Some(row) = rows.next()? { let entry = ScanRange::from_parts( @@ -546,7 +480,7 @@ pub(crate) fn replace_queue_entries( }, ); to_delete_ends.push(Value::from(u32::from(entry.block_range().end))); - existing_ranges = if let Some(cur) = existing_ranges { + to_create = if let Some(cur) = to_create { Some(cur.insert(entry)) } else { Some(SpanningTree::Leaf(entry)) @@ -555,15 +489,12 @@ pub(crate) fn replace_queue_entries( // Update the tree that we read from the database, or if we didn't find any ranges // start with the scanned range. - let mut to_create = match (existing_ranges, entries.next()) { - (Some(cur), Some(entry)) => Some(cur.insert(entry)), - (None, Some(entry)) => Some(SpanningTree::Leaf(entry)), - (Some(cur), None) => Some(cur), - (None, None) => None, - }; - for entry in entries { - to_create = to_create.map(|cur| cur.insert(entry)); + to_create = if let Some(cur) = to_create { + Some(cur.insert(entry)) + } else { + Some(SpanningTree::Leaf(entry)) + }; } (to_create, to_delete_ends) @@ -611,6 +542,16 @@ pub(crate) fn scan_complete( WHERE shard_index = :shard_index", )?; + let mut sapling_shard_end = |index: u64| -> Result, rusqlite::Error> { + Ok(sapling_shard_end_stmt + .query_row(named_params![":shard_index": index], |row| { + row.get::<_, Option>(0) + .map(|opt| opt.map(BlockHeight::from)) + }) + .optional()? + .flatten()) + }; + // if no notes belonging to the wallet were found, so don't need to extend the scanning // range suggestions to include the associated subtrees, and our bounds are just the // scanned range @@ -618,38 +559,18 @@ pub(crate) fn scan_complete( .map(|(min_idx, max_idx)| { let range_min = if *min_idx > 0 { // get the block height of the end of the previous shard - sapling_shard_end_stmt - .query_row(named_params![":shard_index": *min_idx - 1], |row| { - row.get::<_, Option>(0) - .map(|opt| opt.map(BlockHeight::from)) - }) - .optional()? - .flatten() + sapling_shard_end(*min_idx - 1)? } else { // our lower bound is going to be the Sapling activation height params.activation_height(NetworkUpgrade::Sapling) }; // get the block height for the end of the current shard - let range_max = sapling_shard_end_stmt - .query_row(named_params![":shard_index": max_idx], |row| { - row.get::<_, Option>(0) - .map(|opt| opt.map(BlockHeight::from)) - }) - .optional()? - .flatten(); - - Ok::, rusqlite::Error>(match (range_min, range_max) { - (Some(start), Some(end)) => Range { start, end }, - (Some(start), None) => Range { - start, - end: range.end, - }, - (None, Some(end)) => Range { - start: range.start, - end, - }, - (None, None) => range.clone(), + let range_max = sapling_shard_end(*max_idx)?; + + Ok::, rusqlite::Error>(Range { + start: range_min.unwrap_or(range.start), + end: range_max.unwrap_or(range.end), }) }) .transpose() @@ -804,6 +725,10 @@ mod tests { assert_eq!(RangeOrdering::cmp(&(1..2), &(0..1)), RightFirstDisjoint); assert_eq!(RangeOrdering::cmp(&(0..1), &(2..3)), LeftFirstDisjoint); assert_eq!(RangeOrdering::cmp(&(2..3), &(0..1)), RightFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(1..2), &(2..2)), LeftFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(2..2), &(1..2)), RightFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(1..1), &(1..2)), LeftFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(1..2), &(1..1)), RightFirstDisjoint); // Contained assert_eq!(RangeOrdering::cmp(&(1..2), &(0..3)), LeftContained); From cb887efa06c8e4017845a9b92ae205dc2c3b50af Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 13 Jul 2023 00:59:12 +0000 Subject: [PATCH 0064/1122] Use correct levels for cap nodes in `put_shard_roots` The Merkle hashes used for the note commitment trees are domain separated by level, so when pretending that the subtree roots are leaves of the cap tree, we need to adjust for their level not being zero. Closes zcash/librustzcash#874. Co-authored-by: Sean Bowe --- .../src/wallet/commitment_tree.rs | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 12cf243338..892d8c12ac 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -775,9 +775,41 @@ pub(crate) fn put_shard_roots< // We treat the cap as a DEPTH-SHARD_HEIGHT tree so that we can make a batch insertion of // root data using `Position::from(start_index)` as the starting position and treating the // roots as level-0 leaves. + #[derive(Clone, Debug, PartialEq, Eq)] + struct LevelShifter(H); + impl Hashable for LevelShifter { + fn empty_leaf() -> Self { + Self(H::empty_root(SHARD_HEIGHT.into())) + } + + fn combine(level: Level, a: &Self, b: &Self) -> Self { + Self(H::combine(level + SHARD_HEIGHT, &a.0, &b.0)) + } + + fn empty_root(level: Level) -> Self + where + Self: Sized, + { + Self(H::empty_root(level + SHARD_HEIGHT)) + } + } + impl HashSer for LevelShifter { + fn read(reader: R) -> io::Result + where + Self: Sized, + { + H::read(reader).map(Self) + } + + fn write(&self, writer: W) -> io::Result<()> { + self.0.write(writer) + } + } + let cap = LocatedTree::from_parts( Address::from_parts((DEPTH - SHARD_HEIGHT).into(), 0), - get_cap(conn, table_prefix).map_err(ShardTreeError::Storage)?, + get_cap::>(conn, table_prefix) + .map_err(ShardTreeError::Storage)?, ); let cap_result = cap @@ -785,7 +817,7 @@ pub(crate) fn put_shard_roots< Position::from(start_index), roots.iter().map(|r| { ( - r.root_hash().clone(), + LevelShifter(r.root_hash().clone()), Retention::Checkpoint { id: (), is_marked: false, From 281dbd5524b4ef3a6068221473c9f0267c023aa6 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 13 Jul 2023 01:02:27 +0000 Subject: [PATCH 0065/1122] Ensure that `extended_range` in `scan_complete` is a superset of `range` Previously `extended_range` only covered the extent of the leaves of all subtrees in which notes were found during a scan. When the scanned range was large, this was not guaranteed to be contained within the subtree leaves, causing an assertion failure when an invalid `ScanRange` was constructed. --- zcash_client_sqlite/src/wallet/scanning.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index e334781dce..aca6a7b3a9 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -569,8 +569,8 @@ pub(crate) fn scan_complete( let range_max = sapling_shard_end(*max_idx)?; Ok::, rusqlite::Error>(Range { - start: range_min.unwrap_or(range.start), - end: range_max.unwrap_or(range.end), + start: range.start.min(range_min.unwrap_or(range.start)), + end: range.end.max(range_max.unwrap_or(range.end)), }) }) .transpose() From 510944777ca2c1b918c2cd73ad59d41150b4b8fb Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 13 Jul 2023 01:05:18 +0000 Subject: [PATCH 0066/1122] Transactionally modify the wallet DB in `replace_queue_entries` We don't want to delete old scan range queue entries unless we are guaranteed to write the updated queue entries. --- zcash_client_sqlite/src/wallet/scanning.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index aca6a7b3a9..0e7e47a68f 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -426,7 +426,7 @@ pub(crate) fn insert_queue_entries<'a>( } pub(crate) fn replace_queue_entries( - conn: &rusqlite::Connection, + conn: &rusqlite::Transaction<'_>, query_range: &Range, entries: impl Iterator, ) -> Result<(), SqliteClientError> { From 2a98f94f0515dea781828c2eda2c25501947a008 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 13 Jul 2023 01:12:39 +0000 Subject: [PATCH 0067/1122] Add more debug and trace logging The `Debug` impl for `sapling::Node` is updated to output hex-encoded bytes for readability. --- zcash_client_backend/src/data_api/scanning.rs | 18 +++++++++++++++++- zcash_client_sqlite/src/wallet/init.rs | 2 ++ .../init/migrations/shardtree_support.rs | 19 +++++++++++++++++++ zcash_client_sqlite/src/wallet/scanning.rs | 8 ++++++++ zcash_primitives/src/sapling/tree.rs | 12 +++++++++++- 5 files changed, 57 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/src/data_api/scanning.rs b/zcash_client_backend/src/data_api/scanning.rs index c0315f2408..71ab0f8e91 100644 --- a/zcash_client_backend/src/data_api/scanning.rs +++ b/zcash_client_backend/src/data_api/scanning.rs @@ -1,3 +1,4 @@ +use std::fmt; use std::ops::Range; use zcash_primitives::consensus::BlockHeight; @@ -26,10 +27,25 @@ pub struct ScanRange { priority: ScanPriority, } +impl fmt::Display for ScanRange { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{:?}({}..{})", + self.priority, self.block_range.start, self.block_range.end, + ) + } +} + impl ScanRange { /// Constructs a scan range from its constituent parts. pub fn from_parts(block_range: Range, priority: ScanPriority) -> Self { - assert!(block_range.end >= block_range.start); + assert!( + block_range.end >= block_range.start, + "{:?} is invalid for ScanRange({:?})", + block_range, + priority, + ); ScanRange { block_range, priority, diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 5ac93a03f6..013581d81c 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -2,6 +2,7 @@ use either::Either; use incrementalmerkletree::Retention; use std::{collections::HashMap, fmt, io}; +use tracing::debug; use rusqlite::{self, types::ToSql}; use schemer::{Migrator, MigratorError}; @@ -318,6 +319,7 @@ pub fn init_blocks_table( )?; if let Some(nonempty_frontier) = block_end_tree.to_frontier().value() { + debug!("Inserting frontier into ShardTree: {:?}", nonempty_frontier); let shard_store = SqliteShardStore::<_, sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( wdb.conn.0, diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index a00a5b97bb..14a0c20a65 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -9,6 +9,7 @@ use rusqlite::{self, named_params, params}; use schemer; use schemer_rusqlite::RusqliteMigration; use shardtree::ShardTree; +use tracing::{debug, trace}; use uuid::Uuid; use zcash_client_backend::data_api::{ @@ -61,6 +62,7 @@ impl RusqliteMigration for Migration { fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { // Add commitment tree sizes to block metadata. + debug!("Adding new columns"); transaction.execute_batch( "ALTER TABLE blocks ADD COLUMN sapling_commitment_tree_size INTEGER; ALTER TABLE blocks ADD COLUMN orchard_commitment_tree_size INTEGER; @@ -68,6 +70,7 @@ impl RusqliteMigration for Migration { )?; // Add shard persistence + debug!("Creating tables for shard persistence"); transaction.execute_batch( "CREATE TABLE sapling_tree_shards ( shard_index INTEGER PRIMARY KEY, @@ -86,6 +89,7 @@ impl RusqliteMigration for Migration { )?; // Add checkpoint persistence + debug!("Creating tables for checkpoint persistence"); transaction.execute_batch( "CREATE TABLE sapling_tree_checkpoints ( checkpoint_id INTEGER PRIMARY KEY, @@ -133,10 +137,23 @@ impl RusqliteMigration for Migration { ) })?; + if block_height % 1000 == 0 { + debug!(height = block_height, "Migrating tree data to shardtree"); + } + trace!( + height = block_height, + size = block_end_tree.size(), + "Storing Sapling commitment tree size" + ); stmt_update_block_sapling_tree_size .execute(params![block_end_tree.size(), block_height])?; if let Some(nonempty_frontier) = block_end_tree.to_frontier().value() { + trace!( + height = block_height, + frontier = ?nonempty_frontier, + "Inserting frontier nodes", + ); shard_tree.insert_frontier_nodes( nonempty_frontier.clone(), Retention::Checkpoint { @@ -149,6 +166,7 @@ impl RusqliteMigration for Migration { } // Insert all the tree information that we can get from existing incremental witnesses + debug!("Migrating witness data to shardtree"); { let mut stmt_blocks = transaction.prepare("SELECT note, block, witness FROM sapling_witnesses")?; @@ -190,6 +208,7 @@ impl RusqliteMigration for Migration { } // Establish the scan queue & wallet history table + debug!("Creating table for scan queue"); transaction.execute_batch( "CREATE TABLE scan_queue ( block_range_start INTEGER NOT NULL, diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 0e7e47a68f..218d3a648a 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -3,6 +3,7 @@ use std::cmp::{max, min, Ordering}; use std::collections::BTreeSet; use std::ops::{Not, Range}; use std::rc::Rc; +use tracing::{debug, trace}; use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; use incrementalmerkletree::{Address, Position}; @@ -413,6 +414,7 @@ pub(crate) fn insert_queue_entries<'a>( )?; for entry in entries { + trace!("Inserting queue entry {}", entry); if !entry.is_empty() { stmt.execute(named_params![ ":block_range_start": u32::from(entry.block_range().start) , @@ -654,6 +656,12 @@ pub(crate) fn update_chain_tip( } } }); + if let Some(entry) = &shard_entry { + debug!("{} will update latest shard", entry); + } + if let Some(entry) = &tip_entry { + debug!("{} will connect prior tip to new tip", entry); + } let query_range = match (shard_entry.as_ref(), tip_entry.as_ref()) { (Some(se), Some(te)) => Some(Range { diff --git a/zcash_primitives/src/sapling/tree.rs b/zcash_primitives/src/sapling/tree.rs index 5bacb51e23..bd403d6c23 100644 --- a/zcash_primitives/src/sapling/tree.rs +++ b/zcash_primitives/src/sapling/tree.rs @@ -2,6 +2,8 @@ use bitvec::{order::Lsb0, view::AsBits}; use group::{ff::PrimeField, Curve}; use incrementalmerkletree::{Hashable, Level}; use lazy_static::lazy_static; + +use std::fmt; use std::io::{self, Read, Write}; use super::{ @@ -64,11 +66,19 @@ pub fn merkle_hash(depth: usize, lhs: &[u8; 32], rhs: &[u8; 32]) -> [u8; 32] { } /// A node within the Sapling commitment tree. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq)] pub struct Node { pub(super) repr: [u8; 32], } +impl fmt::Debug for Node { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Node") + .field("repr", &hex::encode(self.repr)) + .finish() + } +} + impl Node { #[cfg(test)] pub(crate) fn new(repr: [u8; 32]) -> Self { From a87dca00e25f15f37e000bc211a9b8343200a8ed Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 13 Jul 2023 18:25:56 +0100 Subject: [PATCH 0068/1122] Documentation updates, fixes, and cleanups Co-authored-by: Daira Hopwood --- zcash_client_backend/CHANGELOG.md | 2 + zcash_client_backend/src/data_api.rs | 6 ++- zcash_client_backend/src/data_api/chain.rs | 10 ++-- zcash_client_backend/src/data_api/scanning.rs | 5 +- zcash_client_backend/src/scanning.rs | 2 +- zcash_client_sqlite/src/chain.rs | 8 --- zcash_client_sqlite/src/wallet.rs | 2 +- .../src/wallet/commitment_tree.rs | 6 +-- .../init/migrations/shardtree_support.rs | 3 +- zcash_client_sqlite/src/wallet/scanning.rs | 52 ++++++++++++------- 10 files changed, 54 insertions(+), 42 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index e5c1ae7d77..1d77875dd6 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -85,6 +85,8 @@ and this library adheres to Rust's notion of feature flag, has been modified by the addition of a `sapling_tree` property. - `wallet::input_selection`: - `Proposal::target_height` (use `Proposal::min_target_height` instead). +- `zcash_client_backend::data_api::chain::validate_chain` (logic merged into + `chain::scan_cached_blocks`. - `zcash_client_backend::data_api::chain::error::{ChainError, Cause}` have been replaced by `zcash_client_backend::scanning::ScanError` - `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}` diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 4f4c80c869..ac6118e7c8 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -89,6 +89,8 @@ pub trait WalletRead { /// tree size information for each block; or else the scan is likely to fail if notes belonging /// to the wallet are detected. /// + /// The returned range(s) may include block heights beyond the current chain tip. + /// /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock fn suggest_scan_ranges(&self) -> Result, Self::Error>; @@ -502,8 +504,8 @@ pub trait WalletWrite: WalletRead { /// Updates the wallet's view of the blockchain. /// /// This method is used to provide the wallet with information about the state of the - /// blockchain, and detect any previously scanned that needs to be re-validated before - /// proceeding with scanning. It should be called at wallet startup prior to calling + /// blockchain, and detect any previously scanned data that needs to be re-validated + /// before proceeding with scanning. It should be called at wallet startup prior to calling /// [`WalletRead::suggest_scan_ranges`] in order to provide the wallet with the information it /// needs to correctly prioritize scanning operations. fn update_chain_tip(&mut self, tip_height: BlockHeight) -> Result<(), Self::Error>; diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 54a285eb44..0f6f94545d 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -104,8 +104,7 @@ //! } //! } //! -//! // Truncation will have updated the suggested scan ranges, so we now -//! // re_request +//! // In case we updated the suggested scan ranges, now re-request. //! scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?; //! } //! _ => { @@ -123,10 +122,13 @@ //! // Download the blocks in `scan_range` into the block source. While in this example this //! // step is performed in-line, it's fine for the download of scan ranges to be asynchronous //! // and for the scanner to process the downloaded ranges as they become available in a -//! // separate thread. +//! // separate thread. The scan ranges should also be broken down into smaller chunks as +//! // appropriate, and for ranges with priority `Historic` it can be useful to download and +//! // scan the range in reverse order (to discover more recent unspent notes sooner), or from +//! // the start and end of the range inwards. //! unimplemented!(); //! -//! // Scan the downloaded blocks, +//! // Scan the downloaded blocks. //! let scan_result = scan_cached_blocks( //! &network, //! &block_source, diff --git a/zcash_client_backend/src/data_api/scanning.rs b/zcash_client_backend/src/data_api/scanning.rs index 71ab0f8e91..bb91ba35a5 100644 --- a/zcash_client_backend/src/data_api/scanning.rs +++ b/zcash_client_backend/src/data_api/scanning.rs @@ -10,13 +10,14 @@ pub enum ScanPriority { Scanned, /// Block ranges to be scanned to advance the fully-scanned height. Historic, - /// Block ranges adjacent to wallet open heights. + /// Block ranges adjacent to heights at which the user opened the wallet. OpenAdjacent, /// Blocks that must be scanned to complete note commitment tree shards adjacent to found notes. FoundNote, /// Blocks that must be scanned to complete the latest note commitment tree shard. ChainTip, - /// A previously-scanned range that must be verified has highest priority. + /// A previously scanned range that must be verified to check it is still in the + /// main chain, has highest priority. Verify, } diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index e49f71331c..302cf18432 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -175,7 +175,7 @@ impl fmt::Display for ScanError { at_height ), BlockHeightDiscontinuity { prev_height, new_height } => { - write!(f, "Block height discontinuity at height {}; next height is : {}", prev_height, new_height) + write!(f, "Block height discontinuity at height {}; previous height was: {}", new_height, prev_height) } TreeSizeMismatch { protocol, at_height, given, computed } => { write!(f, "The {:?} note commitment tree size provided by a compact block did not match the expected size at height {}; given {}, expected {}", protocol, at_height, given, computed) diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 9091899754..8c8a1a2723 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -26,10 +26,6 @@ pub mod migrations; /// Starting at `from_height`, the `with_row` callback is invoked with each block retrieved from /// the backing store. If the `limit` value provided is `None`, all blocks are traversed up to the /// maximum height. -/// -/// # Panics -/// -/// Panics if the provided `limit` value exceeds the range of a u32 pub(crate) fn blockdb_with_blocks( block_source: &BlockDb, from_height: Option, @@ -200,10 +196,6 @@ pub(crate) fn blockmetadb_find_block( /// Starting at `from_height`, the `with_row` callback is invoked with each block retrieved from /// the backing store. If the `limit` value provided is `None`, all blocks are traversed up to the /// maximum height for which metadata is available. -/// -/// # Panics -/// -/// Panics if the provided `limit` value exceeds the range of a u32 #[cfg(feature = "unstable")] pub(crate) fn fsblockdb_with_blocks( cache: &FsBlockDb, diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index b90a061072..df71c47742 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -748,7 +748,7 @@ pub(crate) fn truncate_to_height( ) -> Result<(), SqliteClientError> { let sapling_activation_height = params .activation_height(NetworkUpgrade::Sapling) - .expect("Sapling activation height mutst be available."); + .expect("Sapling activation height must be available."); // Recall where we synced up to previously. let last_scanned_height = conn.query_row("SELECT MAX(height) FROM blocks", [], |row| { diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 892d8c12ac..4e551829bf 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -772,9 +772,9 @@ pub(crate) fn put_shard_roots< return Ok(()); } - // We treat the cap as a DEPTH-SHARD_HEIGHT tree so that we can make a batch insertion of - // root data using `Position::from(start_index)` as the starting position and treating the - // roots as level-0 leaves. + // We treat the cap as a tree with `DEPTH - SHARD_HEIGHT` levels, so that we can make a + // batch insertion of root data using `Position::from(start_index)` as the starting position + // and treating the roots as level-0 leaves. #[derive(Clone, Debug, PartialEq, Eq)] struct LevelShifter(H); impl Hashable for LevelShifter { diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 14a0c20a65..3abb29a67f 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -207,7 +207,8 @@ impl RusqliteMigration for Migration { } } - // Establish the scan queue & wallet history table + // Establish the scan queue & wallet history table. + // block_range_end is exclusive. debug!("Creating table for scan queue"); transaction.execute_batch( "CREATE TABLE scan_queue ( diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 218d3a648a..c1ff1c9d8b 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -107,7 +107,8 @@ pub(crate) fn suggest_scan_ranges( // This implements the dominance rule for range priority. If the inserted range's priority is // `Verify`, this replaces any existing priority. Otherwise, if the current priority is -// `Scanned`, this overwrites any priority +// `Scanned`, it remains as `Scanned`; and if the new priority is `Scanned`, it +// overrides any existing priority. fn dominance(current: &ScanPriority, inserted: &ScanPriority, insert: Insert) -> Dominance { match (current.cmp(inserted), (current, inserted)) { (Ordering::Equal, _) => Dominance::Equal, @@ -118,14 +119,25 @@ fn dominance(current: &ScanPriority, inserted: &ScanPriority, insert: Insert) -> } } +/// In the comments for each alternative, `()` represents the left range and `[]` represents the right range. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum RangeOrdering { + /// `( ) [ ]` LeftFirstDisjoint, + /// `( [ ) ]` LeftFirstOverlap, + /// `[ ( ) ]` LeftContained, + /// ```text + /// ( ) + /// [ ] + /// ``` Equal, + /// `( [ ] )` RightContained, + /// `[ ( ] )` RightFirstOverlap, + /// `[ ] ( )` RightFirstDisjoint, } @@ -307,10 +319,11 @@ impl SpanningTree { match self { SpanningTree::Leaf(cur) => Self::from_joined(insert(cur, to_insert)), SpanningTree::Parent { span, left, right } => { - // TODO: this algorithm always preserves the existing partition point, and does not - // do any rebalancing or unification of ranges within the tree; `into_vec` - // performes such unification and the tree being unbalanced should be fine given - // the relatively small number of ranges we should ordinarily be concerned with. + // This algorithm always preserves the existing partition point, and does not do + // any rebalancing or unification of ranges within the tree. This should be okay + // because `into_vec` performs such unification, and the tree being unbalanced + // should be fine given the relatively small number of ranges we should ordinarily + // be concerned with. use RangeOrdering::*; match RangeOrdering::cmp(&span, to_insert.block_range()) { LeftFirstDisjoint => { @@ -417,7 +430,7 @@ pub(crate) fn insert_queue_entries<'a>( trace!("Inserting queue entry {}", entry); if !entry.is_empty() { stmt.execute(named_params![ - ":block_range_start": u32::from(entry.block_range().start) , + ":block_range_start": u32::from(entry.block_range().start), ":block_range_end": u32::from(entry.block_range().end), ":priority": priority_code(&entry.priority()) ])?; @@ -459,7 +472,7 @@ pub(crate) fn replace_queue_entries( ":end": u32::from(query_range.end), ])?; - // Iterate over the ranges in the scan queue that overlaps the range that we have + // Iterate over the ranges in the scan queue that overlap the range that we have // identified as needing to be fully scanned. For each such range add it to the // spanning tree (these should all be nonoverlapping ranges, but we might coalesce // some in the process). @@ -554,9 +567,9 @@ pub(crate) fn scan_complete( .flatten()) }; - // if no notes belonging to the wallet were found, so don't need to extend the scanning + // If no notes belonging to the wallet were found, we don't need to extend the scanning // range suggestions to include the associated subtrees, and our bounds are just the - // scanned range + // scanned range. subtree_bounds .map(|(min_idx, max_idx)| { let range_min = if *min_idx > 0 { @@ -617,13 +630,12 @@ pub(crate) fn update_chain_tip( )?; // Create a scanning range for the fragment of the last shard leading up to new tip. - // However, only do so if the start of the shard is at a stable height. let shard_entry = shard_start_height .filter(|h| h < &chain_end) .map(|h| ScanRange::from_parts(h..chain_end, ScanPriority::ChainTip)); // Create scanning ranges to either validate potentially invalid blocks at the wallet's view - // of the chain tip, + // of the chain tip, or connect the prior tip to the new tip. let tip_entry = block_height_extrema(conn)?.map(|(_, prior_tip)| { // If we don't have shard metadata, this means we're doing linear scanning, so create a // scan range from the prior tip to the current tip with `Historic` priority. @@ -634,7 +646,7 @@ pub(crate) fn update_chain_tip( // and not subject to being reorg'ed. let stable_height = new_tip.saturating_sub(PRUNING_DEPTH); - // if the wallet's prior tip is above the stable height, prioritize the range between + // If the wallet's prior tip is above the stable height, prioritize the range between // it and the new tip as `ChainTip`. Otherwise, prioritize the `VALIDATION_DEPTH` // blocks above the wallet's prior tip as `Verify`. Since `scan_cached_blocks` // retrieves the metadata for the block being connected to, the connectivity to the @@ -954,12 +966,12 @@ mod tests { let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); - // Add an account to the wallet + // Add an account to the wallet. let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); assert_matches!( - // in the following, we don't care what the root hashes are, they just need to be - // distinct + // In the following, we don't care what the root hashes are, they just need to be + // distinct. db_data.put_sapling_subtree_roots( 0, &[ @@ -1021,7 +1033,7 @@ mod tests { Ok(()) ); - // Verify the that adjacent range needed to make the note spendable has been prioritized + // Verify the that adjacent range needed to make the note spendable has been prioritized. let sap_active = u32::from(sapling_activation_height()); assert_matches!( db_data.suggest_scan_ranges(), @@ -1030,7 +1042,7 @@ mod tests { ] ); - // Check that the scanned range has been properly persisted + // Check that the scanned range has been properly persisted. assert_matches!( suggest_scan_ranges(&db_data.conn, Scanned), Ok(scan_ranges) if scan_ranges == vec![ @@ -1039,8 +1051,8 @@ mod tests { ] ); - // simulate the wallet going offline for a bit, update the chain tip to 30 blocks in the - // future + // Simulate the wallet going offline for a bit, update the chain tip to 20 blocks in the + // future. assert_matches!( db_data.update_chain_tip(sapling_activation_height() + 340), Ok(()) @@ -1056,7 +1068,7 @@ mod tests { ] ); - // Now simulate a jump ahead more than 100 blocks + // Now simulate a jump ahead more than 100 blocks. assert_matches!( db_data.update_chain_tip(sapling_activation_height() + 450), Ok(()) From 98ea08e497fba523f165c498983ca05b8807c80e Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 18 Jul 2023 14:27:43 +0000 Subject: [PATCH 0069/1122] Migrate to latest `shardtree` to improve performance --- Cargo.toml | 4 +-- zcash_client_sqlite/src/serialization.rs | 4 +-- .../src/wallet/commitment_tree.rs | 4 +-- .../init/migrations/shardtree_support.rs | 36 ++++++++++++++----- 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 14e9d04239..983e50e242 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,6 @@ panic = 'abort' codegen-units = 1 [patch.crates-io] -incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "67111e29403c33f2e36d6924167f1d5f04ad0fc2" } -shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "67111e29403c33f2e36d6924167f1d5f04ad0fc2" } +incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "bae25ad89c0c192bee625252d2d419bd56638e48" } +shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "bae25ad89c0c192bee625252d2d419bd56638e48" } orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } diff --git a/zcash_client_sqlite/src/serialization.rs b/zcash_client_sqlite/src/serialization.rs index eb11764659..a847d8672f 100644 --- a/zcash_client_sqlite/src/serialization.rs +++ b/zcash_client_sqlite/src/serialization.rs @@ -4,7 +4,7 @@ use byteorder::{ReadBytesExt, WriteBytesExt}; use core::ops::Deref; use shardtree::{Node, PrunableTree, RetentionFlags, Tree}; use std::io::{self, Read, Write}; -use std::rc::Rc; +use std::sync::Arc; use zcash_encoding::Optional; use zcash_primitives::merkle_tree::HashSer; @@ -53,7 +53,7 @@ pub fn write_shard(writer: &mut W, tree: &PrunableTree) fn read_shard_v1(mut reader: &mut R) -> io::Result> { match reader.read_u8()? { PARENT_TAG => { - let ann = Optional::read(&mut reader, ::read)?.map(Rc::new); + let ann = Optional::read(&mut reader, ::read)?.map(Arc::new); let left = read_shard_v1(reader)?; let right = read_shard_v1(reader)?; Ok(Tree::parent(ann, left, right)) diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 4e551829bf..be5c38b42b 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -4,7 +4,7 @@ use std::{ collections::BTreeSet, io::{self, Cursor}, marker::PhantomData, - rc::Rc, + sync::Arc, }; use zcash_client_backend::data_api::chain::CommitmentTreeRoot; @@ -281,7 +281,7 @@ pub(crate) fn get_shard( let located_tree = LocatedPrunableTree::from_parts(shard_root_addr, shard_tree); if let Some(root_hash_data) = root_hash { let root_hash = H::read(Cursor::new(root_hash_data)).map_err(Either::Left)?; - Ok(located_tree.reannotate_root(Some(Rc::new(root_hash)))) + Ok(located_tree.reannotate_root(Some(Arc::new(root_hash)))) } else { Ok(located_tree) } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 3abb29a67f..f8c4a61af1 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -8,7 +8,7 @@ use incrementalmerkletree::Retention; use rusqlite::{self, named_params, params}; use schemer; use schemer_rusqlite::RusqliteMigration; -use shardtree::ShardTree; +use shardtree::{caching::CachingShardStore, ShardTree, ShardTreeError}; use tracing::{debug, trace}; use uuid::Uuid; @@ -108,6 +108,7 @@ impl RusqliteMigration for Migration { transaction, SAPLING_TABLES_PREFIX, )?; + let shard_store = CachingShardStore::load(shard_store).map_err(ShardTreeError::Storage)?; let mut shard_tree: ShardTree< _, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, @@ -154,13 +155,19 @@ impl RusqliteMigration for Migration { frontier = ?nonempty_frontier, "Inserting frontier nodes", ); - shard_tree.insert_frontier_nodes( - nonempty_frontier.clone(), - Retention::Checkpoint { - id: BlockHeight::from(block_height), - is_marked: false, - }, - )?; + shard_tree + .insert_frontier_nodes( + nonempty_frontier.clone(), + Retention::Checkpoint { + id: BlockHeight::from(block_height), + is_marked: false, + }, + ) + .map_err(|e| match e { + ShardTreeError::Query(e) => ShardTreeError::Query(e), + ShardTreeError::Insert(e) => ShardTreeError::Insert(e), + ShardTreeError::Storage(_) => unreachable!(), + })? } } } @@ -203,10 +210,21 @@ impl RusqliteMigration for Migration { updated_note_positions.insert(witnessed_position); } - shard_tree.insert_witness_nodes(witness, BlockHeight::from(block_height))?; + shard_tree + .insert_witness_nodes(witness, BlockHeight::from(block_height)) + .map_err(|e| match e { + ShardTreeError::Query(e) => ShardTreeError::Query(e), + ShardTreeError::Insert(e) => ShardTreeError::Insert(e), + ShardTreeError::Storage(_) => unreachable!(), + })?; } } + shard_tree + .into_store() + .flush() + .map_err(ShardTreeError::Storage)?; + // Establish the scan queue & wallet history table. // block_range_end is exclusive. debug!("Creating table for scan queue"); From 359ff55ce3854c044d5be9a6038fc29191ad20d1 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 18 Jul 2023 18:29:52 +0000 Subject: [PATCH 0070/1122] Do nothing if `update_chain_tip` is given a tip lower than we can use Previously it would panic due to constructing invalid `ScanRange`s. --- zcash_client_sqlite/src/wallet/scanning.rs | 33 +++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index c1ff1c9d8b..e3d8901b7e 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -618,6 +618,27 @@ pub(crate) fn update_chain_tip( params: &P, new_tip: BlockHeight, ) -> Result<(), SqliteClientError> { + // Read the previous tip height from the blocks table. + let prior_tip = block_height_extrema(conn)?.map(|(_, prior_tip)| prior_tip); + + // If the chain tip is below the prior tip height, then the caller has caught the + // chain in the middle of a reorg. Do nothing; the caller will continue using the old + // scan ranges and either: + // - encounter an error trying to fetch the blocks (and thus trigger the same handling + // logic as if this happened with the old linear scanning code); or + // - encounter a discontinuity error in `scan_cached_blocks`, at which point they will + // call `WalletDb::truncate_to_height` as part of their reorg handling which will + // resolve the problem. + // + // We don't check the shard height, as normal usage would have the caller update the + // shard state prior to this call, so it is possible and expected to be in a situation + // where we should update the tip-related scan ranges but not the shard-related ones. + if let Some(h) = prior_tip { + if new_tip < h { + return Ok(()); + } + } + // `ScanRange` uses an exclusive upper bound. let chain_end = new_tip + 1; @@ -636,7 +657,7 @@ pub(crate) fn update_chain_tip( // Create scanning ranges to either validate potentially invalid blocks at the wallet's view // of the chain tip, or connect the prior tip to the new tip. - let tip_entry = block_height_extrema(conn)?.map(|(_, prior_tip)| { + let tip_entry = prior_tip.map(|prior_tip| { // If we don't have shard metadata, this means we're doing linear scanning, so create a // scan range from the prior tip to the current tip with `Historic` priority. if shard_entry.is_none() { @@ -696,9 +717,13 @@ pub(crate) fn update_chain_tip( // have no existing scan queue entries and can fall back to linear scanning from Sapling // activation. if let Some(sapling_activation) = params.activation_height(NetworkUpgrade::Sapling) { - let scan_range = - ScanRange::from_parts(sapling_activation..chain_end, ScanPriority::Historic); - insert_queue_entries(conn, Some(scan_range).iter())?; + // If the caller provided a chain tip that is before Sapling activation, do + // nothing. + if sapling_activation < chain_end { + let scan_range = + ScanRange::from_parts(sapling_activation..chain_end, ScanPriority::Historic); + insert_queue_entries(conn, Some(scan_range).iter())?; + } } } From e16aa41117e634a62b9f7b9118eb8945e2aa6ec0 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Tue, 18 Jul 2023 13:00:57 -0600 Subject: [PATCH 0071/1122] Add test for join_nonoverlapping. --- zcash_client_sqlite/src/wallet/scanning.rs | 104 ++++++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index e3d8901b7e..2c29f437d3 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -157,6 +157,7 @@ impl RangeOrdering { } } +#[derive(Debug, PartialEq, Eq)] enum Joined { One(ScanRange), Two(ScanRange, ScanRange), @@ -757,7 +758,108 @@ mod tests { BlockDb, WalletDb, }; - use super::{RangeOrdering, SpanningTree}; + use super::{join_nonoverlapping, Joined, RangeOrdering, SpanningTree}; + + #[test] + fn test_join_nonoverlapping() { + fn test_range(left: ScanRange, right: ScanRange, expected_joined: Joined) { + let joined = join_nonoverlapping(left, right); + + assert_eq!(joined, expected_joined); + } + + macro_rules! range { + ( $start:expr, $end:expr; $priority:ident ) => { + ScanRange::from_parts( + BlockHeight::from($start)..BlockHeight::from($end), + ScanPriority::$priority, + ) + }; + } + + macro_rules! joined { + ( + ($a_start:expr, $a_end:expr; $a_priority:ident) + ) => { + Joined::One( + range!($a_start, $a_end; $a_priority) + ) + }; + ( + ($a_start:expr, $a_end:expr; $a_priority:ident), + ($b_start:expr, $b_end:expr; $b_priority:ident) + ) => { + Joined::Two( + range!($a_start, $a_end; $a_priority), + range!($b_start, $b_end; $b_priority) + ) + }; + ( + ($a_start:expr, $a_end:expr; $a_priority:ident), + ($b_start:expr, $b_end:expr; $b_priority:ident), + ($c_start:expr, $c_end:expr; $c_priority:ident) + + ) => { + Joined::Three( + range!($a_start, $a_end; $a_priority), + range!($b_start, $b_end; $b_priority), + range!($c_start, $c_end; $c_priority) + ) + }; + } + + // Scan ranges have the same priority and + // line up. + test_range( + range!(1, 9; OpenAdjacent), + range!(9, 15; OpenAdjacent), + joined!( + (1, 15; OpenAdjacent) + ), + ); + + // Scan ranges have different priorities, + // so we cannot merge them even though they + // line up. + test_range( + range!(1, 9; OpenAdjacent), + range!(9, 15; ChainTip), + joined!( + (1, 9; OpenAdjacent), + (9, 15; ChainTip) + ), + ); + + // Scan ranges have the same priority but + // do not line up. + test_range( + range!(1, 9; OpenAdjacent), + range!(13, 15; OpenAdjacent), + joined!( + (1, 9; OpenAdjacent), + (9, 13; Historic), + (13, 15; OpenAdjacent) + ), + ); + + test_range( + range!(1, 9; Historic), + range!(13, 15; OpenAdjacent), + joined!( + (1, 13; Historic), + (13, 15; OpenAdjacent) + ), + ); + + test_range( + range!(1, 9; OpenAdjacent), + range!(13, 15; Historic), + joined!( + (1, 9; OpenAdjacent), + (9, 15; Historic) + ), + ); + } #[test] fn range_ordering() { From c7b308b3129294da0b26616f774c1ce892fb2ac6 Mon Sep 17 00:00:00 2001 From: str4d Date: Wed, 19 Jul 2023 17:02:28 +0100 Subject: [PATCH 0072/1122] Rename `VALIDATION_DEPTH` constant to `VERIFY_LOOKAHEAD` Co-authored-by: Daira Emma Hopwood --- zcash_client_sqlite/src/lib.rs | 4 ++-- zcash_client_sqlite/src/wallet.rs | 4 ++-- zcash_client_sqlite/src/wallet/scanning.rs | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 94fb88f57a..7dad65e907 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -89,8 +89,8 @@ use wallet::commitment_tree::put_shard_roots; /// this delta from the chain tip to be pruned. pub(crate) const PRUNING_DEPTH: u32 = 100; -/// The number of blocks to re-verify when the chain tip is updated. -pub(crate) const VALIDATION_DEPTH: u32 = 10; +/// The number of blocks to verify ahead when the chain tip is updated. +pub(crate) const VERIFY_LOOKAHEAD: u32 = 10; pub(crate) const SAPLING_TABLES_PREFIX: &str = "sapling"; diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index df71c47742..285ba352b8 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -93,7 +93,7 @@ use zcash_client_backend::{ wallet::WalletTx, }; -use crate::VALIDATION_DEPTH; +use crate::VERIFY_LOOKAHEAD; use crate::{ error::SqliteClientError, SqlTransaction, WalletCommitmentTrees, WalletDb, PRUNING_DEPTH, }; @@ -834,7 +834,7 @@ pub(crate) fn truncate_to_height( )?; // Prioritize the range starting at the height we just rewound to for verification - let query_range = block_height..(block_height + VALIDATION_DEPTH); + let query_range = block_height..(block_height + VERIFY_LOOKAHEAD); let scan_range = ScanRange::from_parts(query_range.clone(), ScanPriority::Verify); replace_queue_entries(conn, &query_range, Some(scan_range).into_iter())?; } diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 2c29f437d3..d3f456e1ab 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -12,7 +12,7 @@ use zcash_primitives::consensus::{self, BlockHeight, NetworkUpgrade}; use zcash_client_backend::data_api::SAPLING_SHARD_HEIGHT; use crate::error::SqliteClientError; -use crate::{PRUNING_DEPTH, VALIDATION_DEPTH}; +use crate::{PRUNING_DEPTH, VERIFY_LOOKAHEAD}; use super::block_height_extrema; @@ -669,7 +669,7 @@ pub(crate) fn update_chain_tip( let stable_height = new_tip.saturating_sub(PRUNING_DEPTH); // If the wallet's prior tip is above the stable height, prioritize the range between - // it and the new tip as `ChainTip`. Otherwise, prioritize the `VALIDATION_DEPTH` + // it and the new tip as `ChainTip`. Otherwise, prioritize the `VERIFY_LOOKAHEAD` // blocks above the wallet's prior tip as `Verify`. Since `scan_cached_blocks` // retrieves the metadata for the block being connected to, the connectivity to the // prior tip will always be checked. Since `Verify` ranges have maximum priority, even @@ -684,7 +684,7 @@ pub(crate) fn update_chain_tip( // and advance it up to at most the stable height. The shard entry will then cover // the range to the new tip at the lower `ChainTip` priority. ScanRange::from_parts( - prior_tip..min(stable_height, prior_tip + VALIDATION_DEPTH), + prior_tip..min(stable_height, prior_tip + VERIFY_LOOKAHEAD), ScanPriority::Verify, ) } From 963496d0abda5c19df1b770593771aab86c8fb57 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 19 Jul 2023 16:38:03 +0000 Subject: [PATCH 0073/1122] Use an upsert for `FsBlockDb::write_block_metadata` --- zcash_client_sqlite/CHANGELOG.md | 2 ++ zcash_client_sqlite/src/chain.rs | 35 ++++++++++++++++++++++++-------- zcash_client_sqlite/src/lib.rs | 3 ++- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index ee38993f37..91f75a523d 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -20,6 +20,8 @@ and this library adheres to Rust's notion of wallet did not contain enough observed blocks to satisfy the `min_confirmations` value specified; this situation is now treated as an error. - A `BlockConflict` variant has been added to `zcash_client_sqlite::error::SqliteClientError` +- `zcash_client_sqlite::FsBlockDb::write_block_metadata` now overwrites any + existing metadata entries that have the same height as a new entry. ### Removed - The empty `wallet::transact` module has been removed. diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 8c8a1a2723..2cf51c8277 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -103,21 +103,40 @@ pub(crate) fn blockmetadb_insert( conn: &Connection, block_meta: &[BlockMeta], ) -> Result<(), rusqlite::Error> { + use rusqlite::named_params; + let mut stmt_insert = conn.prepare( - "INSERT INTO compactblocks_meta (height, blockhash, time, sapling_outputs_count, orchard_actions_count) - VALUES (?, ?, ?, ?, ?)" + "INSERT INTO compactblocks_meta ( + height, + blockhash, + time, + sapling_outputs_count, + orchard_actions_count + ) + VALUES ( + :height, + :blockhash, + :time, + :sapling_outputs_count, + :orchard_actions_count + ) + ON CONFLICT (height) DO UPDATE + SET blockhash = :blockhash, + time = :time, + sapling_outputs_count = :sapling_outputs_count, + orchard_actions_count = :orchard_actions_count", )?; conn.execute("BEGIN IMMEDIATE", [])?; let result = block_meta .iter() .map(|m| { - stmt_insert.execute(params![ - u32::from(m.height), - &m.block_hash.0[..], - m.block_time, - m.sapling_outputs_count, - m.orchard_actions_count, + stmt_insert.execute(named_params![ + ":height": u32::from(m.height), + ":blockhash": &m.block_hash.0[..], + ":time": m.block_time, + ":sapling_outputs_count": m.sapling_outputs_count, + ":orchard_actions_count": m.orchard_actions_count, ]) }) .collect::, _>>(); diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 7dad65e907..eb28c59638 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -907,7 +907,8 @@ impl FsBlockDb { Ok(chain::blockmetadb_get_max_cached_height(&self.conn)?) } - /// Adds a set of block metadata entries to the metadata database. + /// Adds a set of block metadata entries to the metadata database, overwriting any + /// existing entries at the given block heights. /// /// This will return an error if any block file corresponding to one of these metadata records /// is absent from the blocks directory. From fab68d217e7cac81bd3e8dd102f4ba4768a59442 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 21 Jul 2023 15:23:49 +0000 Subject: [PATCH 0074/1122] zcash_client_sqlite: Fix `WalletDb::block_fully_scanned` implementation --- zcash_client_sqlite/src/wallet.rs | 32 ++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 285ba352b8..d30005ba0c 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -659,13 +659,35 @@ pub(crate) fn block_metadata( pub(crate) fn block_fully_scanned( conn: &rusqlite::Connection, ) -> Result, SqliteClientError> { - // FIXME: this will need to be rewritten once out-of-order scan range suggestion - // is implemented. + // We assume here that the wallet was either initialized via `init_blocks_table`, or + // its birthday is Sapling activation, so the earliest block in the `blocks` table is + // the first fully-scanned block (because it occurs before any wallet activity). + // + // We further assume that the only way we get a contiguous range of block heights in + // the `blocks` table starting with this earliest block, is if all scanning operations + // have been performed on those blocks. This holds because the `blocks` table is only + // altered by `WalletDb::put_blocks` via `put_block`, and the effective combination of + // intra-range linear scanning and the nullifier map ensures that we discover all + // wallet-related information within the contiguous range. + // + // The fully-scanned height is therefore the greatest height in the first contiguous + // range of block rows, which is a combined case of the "gaps and islands" and + // "greatest N per group" SQL query problems. conn.query_row( "SELECT height, hash, sapling_commitment_tree_size, sapling_tree - FROM blocks - ORDER BY height DESC - LIMIT 1", + FROM blocks + INNER JOIN ( + WITH contiguous AS ( + SELECT height, ROW_NUMBER() OVER (ORDER BY height) - height AS grp + FROM blocks + ) + SELECT MAX(height) AS [fully_scanned_height] + FROM contiguous + GROUP BY grp + ORDER BY height + LIMIT 1 + ) + ON height = fully_scanned_height", [], |row| { let height: u32 = row.get(0)?; From 4d2abd5c3aae93a1fedb181d92de207d1db9b318 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 21 Jul 2023 17:53:14 +0000 Subject: [PATCH 0075/1122] zcash_client_sqlite: Add test showing out-of-order balance problem --- zcash_client_sqlite/src/chain.rs | 81 ++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 2cf51c8277..7a2396dfe1 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -835,4 +835,85 @@ mod tests { (value - value2).unwrap() ); } + + #[test] + fn scan_cached_blocks_detects_spends_out_of_order() { + let cache_file = NamedTempFile::new().unwrap(); + let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); + init_cache_database(&db_cache).unwrap(); + + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); + + // Add an account to the wallet + let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); + + // Account balance should be zero + assert_eq!( + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + Amount::zero() + ); + + // Create a fake CompactBlock sending value to the address + let value = Amount::from_u64(5).unwrap(); + let (cb, nf) = fake_compact_block( + sapling_activation_height(), + BlockHash([0; 32]), + &dfvk, + AddressType::DefaultExternal, + value, + 0, + ); + insert_into_cache(&db_cache, &cb); + + // Create a second fake CompactBlock spending value from the address + let extsk2 = ExtendedSpendingKey::master(&[0]); + let to2 = extsk2.default_address().1; + let value2 = Amount::from_u64(2).unwrap(); + insert_into_cache( + &db_cache, + &fake_compact_block_spending( + sapling_activation_height() + 1, + cb.hash(), + (nf, value), + &dfvk, + to2, + value2, + 1, + ), + ); + + // Scan the spending block first. + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height() + 1, + 1, + ) + .unwrap(); + + // Account balance should equal the change + assert_eq!( + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + (value - value2).unwrap() + ); + + // Now scan the block in which we received the note that was spent. + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 1, + ) + .unwrap(); + + // Account balance should be the same. + assert_eq!( + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + (value - value2).unwrap() + ); + } } From 0f2689b9c3b71341ff1b32000c0af1169e6ae9c4 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 21 Jul 2023 18:01:52 +0000 Subject: [PATCH 0076/1122] zcash_client_sqlite: Maintain a nullifier map from out-of-order scanning Closes zcash/librustzcash#876. --- zcash_client_backend/src/data_api.rs | 9 +- zcash_client_backend/src/scanning.rs | 59 ++++--- zcash_client_sqlite/src/lib.rs | 34 +++- zcash_client_sqlite/src/wallet.rs | 167 ++++++++++++++++++ zcash_client_sqlite/src/wallet/init.rs | 18 ++ .../src/wallet/init/migrations.rs | 6 + .../wallet/init/migrations/nullifier_map.rs | 72 ++++++++ zcash_client_sqlite/src/wallet/sapling.rs | 6 +- 8 files changed, 340 insertions(+), 31 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/nullifier_map.rs diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index ac6118e7c8..8b27ca3df8 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -300,6 +300,7 @@ pub struct ScannedBlock { metadata: BlockMetadata, block_time: u32, transactions: Vec>, + sapling_nullifier_map: Vec<(TxId, u16, Vec)>, sapling_commitments: Vec<(sapling::Node, Retention)>, } @@ -308,12 +309,14 @@ impl ScannedBlock { metadata: BlockMetadata, block_time: u32, transactions: Vec>, + sapling_nullifier_map: Vec<(TxId, u16, Vec)>, sapling_commitments: Vec<(sapling::Node, Retention)>, ) -> Self { Self { metadata, block_time, transactions, + sapling_nullifier_map, sapling_commitments, } } @@ -338,6 +341,10 @@ impl ScannedBlock { &self.transactions } + pub fn sapling_nullifier_map(&self) -> &[(TxId, u16, Vec)] { + &self.sapling_nullifier_map + } + pub fn sapling_commitments(&self) -> &[(sapling::Node, Retention)] { &self.sapling_commitments } @@ -498,7 +505,7 @@ pub trait WalletWrite: WalletRead { /// `blocks` must be sequential, in order of increasing block height fn put_blocks( &mut self, - block: Vec>, + blocks: Vec>, ) -> Result, Self::Error>; /// Updates the wallet's view of the blockchain. diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index 302cf18432..0f25571f99 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -355,36 +355,44 @@ pub(crate) fn scan_block_with_runner< let compact_block_tx_count = block.vtx.len(); let mut wtxs: Vec> = vec![]; + let mut sapling_nullifier_map = Vec::with_capacity(block.vtx.len()); let mut sapling_note_commitments: Vec<(sapling::Node, Retention)> = vec![]; for (tx_idx, tx) in block.vtx.into_iter().enumerate() { let txid = tx.txid(); + let tx_index = + u16::try_from(tx.index).expect("Cannot fit more than 2^16 transactions in a block"); - // Check for spent notes. The only step that is not constant-time is - // the filter() at the end. + // Check for spent notes. The comparison against known-unspent nullifiers is done + // in constant time. // TODO: However, this is O(|nullifiers| * |notes|); does using // constant-time operations here really make sense? - let shielded_spends: Vec<_> = tx - .spends - .into_iter() - .enumerate() - .map(|(index, spend)| { - let spend_nf = spend.nf().expect( - "Could not deserialize nullifier for spend from protobuf representation.", - ); - // Find the first tracked nullifier that matches this spend, and produce - // a WalletShieldedSpend if there is a match, in constant time. - nullifiers - .iter() - .map(|&(account, nf)| CtOption::new(account, nf.ct_eq(&spend_nf))) - .fold( - CtOption::new(AccountId::from(0), 0.into()), - |first, next| CtOption::conditional_select(&next, &first, first.is_some()), - ) - .map(|account| WalletSaplingSpend::from_parts(index, spend_nf, account)) - }) - .filter(|spend| spend.is_some().into()) - .map(|spend| spend.unwrap()) - .collect(); + let mut shielded_spends = vec![]; + let mut sapling_unlinked_nullifiers = Vec::with_capacity(tx.spends.len()); + for (index, spend) in tx.spends.into_iter().enumerate() { + let spend_nf = spend + .nf() + .expect("Could not deserialize nullifier for spend from protobuf representation."); + + // Find the first tracked nullifier that matches this spend, and produce + // a WalletShieldedSpend if there is a match, in constant time. + let spend = nullifiers + .iter() + .map(|&(account, nf)| CtOption::new(account, nf.ct_eq(&spend_nf))) + .fold( + CtOption::new(AccountId::from(0), 0.into()), + |first, next| CtOption::conditional_select(&next, &first, first.is_some()), + ) + .map(|account| WalletSaplingSpend::from_parts(index, spend_nf, account)); + + if spend.is_some().into() { + shielded_spends.push(spend.unwrap()); + } else { + // This nullifier didn't match any we are currently tracking; save it in + // case it matches an earlier block range we haven't scanned yet. + sapling_unlinked_nullifiers.push(spend_nf); + } + } + sapling_nullifier_map.push((txid, tx_index, sapling_unlinked_nullifiers)); // Collect the set of accounts that were spent from in this transaction let spent_from_accounts: HashSet<_> = shielded_spends @@ -505,7 +513,7 @@ pub(crate) fn scan_block_with_runner< if !(shielded_spends.is_empty() && shielded_outputs.is_empty()) { wtxs.push(WalletTx { txid, - index: tx.index as usize, + index: tx_index as usize, sapling_spends: shielded_spends, sapling_outputs: shielded_outputs, }); @@ -518,6 +526,7 @@ pub(crate) fn scan_block_with_runner< BlockMetadata::from_parts(cur_height, cur_hash, sapling_commitment_tree_size), block.time, wtxs, + sapling_nullifier_map, sapling_note_commitments, )) } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index eb28c59638..425dfe54e6 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -442,14 +442,31 @@ impl WalletWrite for WalletDb } for output in &tx.sapling_outputs { - let received_note_id = - wallet::sapling::put_received_note(wdb.conn.0, output, tx_row)?; + // Check whether this note was spent in a later block range that + // we previously scanned. + let spent_in = wallet::query_nullifier_map( + wdb.conn.0, + ShieldedProtocol::Sapling, + output.nf(), + )?; + + let received_note_id = wallet::sapling::put_received_note( + wdb.conn.0, output, tx_row, spent_in, + )?; // Save witness for note. wallet_note_ids.push(received_note_id); } } + // Insert the new nullifiers from this block into the nullifier map. + wallet::insert_nullifier_map( + wdb.conn.0, + block.height(), + ShieldedProtocol::Sapling, + block.sapling_nullifier_map(), + )?; + note_positions.extend(block.transactions().iter().flat_map(|wtx| { wtx.sapling_outputs .iter() @@ -460,6 +477,14 @@ impl WalletWrite for WalletDb sapling_commitments.extend(block.into_sapling_commitments().into_iter()); } + // Prune the nullifier map of entries we no longer need. + if let Some(meta) = wdb.block_fully_scanned()? { + wallet::prune_nullifier_map( + wdb.conn.0, + meta.block_height().saturating_sub(PRUNING_DEPTH), + )?; + } + // We will have a start position and a last scanned height in all cases where // `blocks` is non-empty. if let Some(((start_height, start_position), last_scanned_height)) = @@ -533,7 +558,7 @@ impl WalletWrite for WalletDb )?; if matches!(recipient, Recipient::InternalAccount(_, _)) { - wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; + wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref, None)?; } } TransferType::Incoming => { @@ -548,7 +573,7 @@ impl WalletWrite for WalletDb } } - wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; + wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref, None)?; } } } @@ -645,6 +670,7 @@ impl WalletWrite for WalletDb transfer_type: TransferType::WalletInternal, }, tx_ref, + None, )?; } } diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index d30005ba0c..23704cdd35 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -839,6 +839,14 @@ pub(crate) fn truncate_to_height( [u32::from(block_height)], )?; + // Delete from the nullifier map any entries with a locator referencing a block + // height greater than the truncation height. + conn.execute( + "DELETE FROM tx_locator_map + WHERE block_height > :block_height", + named_params![":block_height": u32::from(block_height)], + )?; + // Delete from the scanning queue any range with a start height greater than the // truncation height, and then truncate any remaining range by setting the end // equal to the truncation height + 1. @@ -1316,6 +1324,165 @@ pub(crate) fn put_sent_output( Ok(()) } +/// Inserts the given entries into the nullifier map. +/// +/// Returns an error if the new entries conflict with existing ones. This indicates either +/// corrupted data, or that a reorg has occurred and the caller needs to repair the wallet +/// state with [`truncate_to_height`]. +pub(crate) fn insert_nullifier_map>( + conn: &rusqlite::Transaction<'_>, + block_height: BlockHeight, + spend_pool: ShieldedProtocol, + new_entries: &[(TxId, u16, Vec)], +) -> Result<(), SqliteClientError> { + let mut stmt_select_tx_locators = conn.prepare_cached( + "SELECT block_height, tx_index, txid + FROM tx_locator_map + WHERE (block_height = :block_height AND tx_index = :tx_index) OR txid = :txid", + )?; + let mut stmt_insert_tx_locator = conn.prepare_cached( + "INSERT INTO tx_locator_map + (block_height, tx_index, txid) + VALUES (:block_height, :tx_index, :txid)", + )?; + let mut stmt_insert_nullifier_mapping = conn.prepare_cached( + "INSERT INTO nullifier_map + (spend_pool, nf, block_height, tx_index) + VALUES (:spend_pool, :nf, :block_height, :tx_index) + ON CONFLICT (spend_pool, nf) DO UPDATE + SET block_height = :block_height, + tx_index = :tx_index", + )?; + + for (txid, tx_index, nullifiers) in new_entries { + let tx_args = named_params![ + ":block_height": u32::from(block_height), + ":tx_index": tx_index, + ":txid": txid.as_ref(), + ]; + + // We cannot use an upsert here, because we use the tx locator as the foreign key + // in `nullifier_map` instead of `txid` for database size efficiency. If an insert + // into `tx_locator_map` were to conflict, we would need the resulting update to + // cascade into `nullifier_map` as either: + // - an update (if a transaction moved within a block), or + // - a deletion (if the locator now points to a different transaction). + // + // `ON UPDATE` has `CASCADE` to always update, but has no deletion option. So we + // instead set `ON UPDATE RESTRICT` on the foreign key relation, and require the + // caller to manually rewind the database in this situation. + let locator = stmt_select_tx_locators + .query_map(tx_args, |row| { + Ok(( + BlockHeight::from_u32(row.get(0)?), + row.get::<_, u16>(1)?, + TxId::from_bytes(row.get(2)?), + )) + })? + .fold(Ok(None), |acc: Result<_, SqliteClientError>, row| { + match (acc?, row?) { + (None, rhs) => Ok(Some(Some(rhs))), + // If there was more than one row, then due to the uniqueness + // constraints on the `tx_locator_map` table, all of the rows conflict + // with the locator being inserted. + (Some(_), _) => Ok(Some(None)), + } + })?; + + match locator { + // If the locator in the table matches the one being inserted, do nothing. + Some(Some(loc)) if loc == (block_height, *tx_index, *txid) => (), + // If the locator being inserted would conflict, report it. + Some(_) => Err(SqliteClientError::DbError(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_CONSTRAINT), + Some("UNIQUE constraint failed: tx_locator_map.block_height, tx_locator_map.tx_index".into()), + )))?, + // If the locator doesn't exist, insert it. + None => stmt_insert_tx_locator.execute(tx_args).map(|_| ())?, + } + + for nf in nullifiers { + // Here it is okay to use an upsert, because per above we've confirmed that + // the locator points to the same transaction. + let nf_args = named_params![ + ":spend_pool": pool_code(PoolType::Shielded(spend_pool)), + ":nf": nf.as_ref(), + ":block_height": u32::from(block_height), + ":tx_index": tx_index, + ]; + stmt_insert_nullifier_mapping.execute(nf_args)?; + } + } + + Ok(()) +} + +/// Returns the row of the `transactions` table corresponding to the transaction in which +/// this nullifier is revealed, if any. +pub(crate) fn query_nullifier_map>( + conn: &rusqlite::Transaction<'_>, + spend_pool: ShieldedProtocol, + nf: &N, +) -> Result, SqliteClientError> { + let mut stmt_select_locator = conn.prepare_cached( + "SELECT block_height, tx_index, txid + FROM nullifier_map + LEFT JOIN tx_locator_map USING (block_height, tx_index) + WHERE spend_pool = :spend_pool AND nf = :nf", + )?; + + let sql_args = named_params![ + ":spend_pool": pool_code(PoolType::Shielded(spend_pool)), + ":nf": nf.as_ref(), + ]; + + // Find the locator corresponding to this nullifier, if any. + let locator = stmt_select_locator + .query_row(sql_args, |row| { + Ok(( + BlockHeight::from_u32(row.get(0)?), + row.get(1)?, + TxId::from_bytes(row.get(2)?), + )) + }) + .optional()?; + let (height, index, txid) = match locator { + Some(res) => res, + None => return Ok(None), + }; + + // Find or create a corresponding row in the `transactions` table. Usually a row will + // have been created during the same scan that the locator was added to the nullifier + // map, but it would not happen if the transaction in question spent the note with no + // change or explicit in-wallet recipient. + put_tx_meta( + conn, + &WalletTx:: { + txid, + index, + sapling_spends: vec![], + sapling_outputs: vec![], + }, + height, + ) + .map(Some) +} + +/// Deletes from the nullifier map any entries with a locator referencing a block height +/// lower than the pruning height. +pub(crate) fn prune_nullifier_map( + conn: &rusqlite::Transaction<'_>, + block_height: BlockHeight, +) -> Result<(), SqliteClientError> { + conn.execute( + "DELETE FROM tx_locator_map + WHERE block_height < :block_height", + named_params![":block_height": u32::from(block_height)], + )?; + + Ok(()) +} + #[cfg(test)] mod tests { use std::num::NonZeroU32; diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 013581d81c..c5d9ab7639 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -414,6 +414,18 @@ mod tests { sapling_tree BLOB NOT NULL , sapling_commitment_tree_size INTEGER, orchard_commitment_tree_size INTEGER)", + "CREATE TABLE nullifier_map ( + spend_pool INTEGER NOT NULL, + nf BLOB NOT NULL, + block_height INTEGER NOT NULL, + tx_index INTEGER NOT NULL, + CONSTRAINT tx_locator + FOREIGN KEY (block_height, tx_index) + REFERENCES tx_locator_map(block_height, tx_index) + ON DELETE CASCADE + ON UPDATE RESTRICT, + CONSTRAINT nf_uniq UNIQUE (spend_pool, nf) + )", "CREATE TABLE sapling_received_notes ( id_note INTEGER PRIMARY KEY, tx INTEGER NOT NULL, @@ -507,6 +519,12 @@ mod tests { fee INTEGER, FOREIGN KEY (block) REFERENCES blocks(height) )", + "CREATE TABLE tx_locator_map ( + block_height INTEGER NOT NULL, + tx_index INTEGER NOT NULL, + txid BLOB NOT NULL UNIQUE, + PRIMARY KEY (block_height, tx_index) + )", "CREATE TABLE \"utxos\" ( id_utxo INTEGER PRIMARY KEY, received_by_account INTEGER NOT NULL, diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index e51605ccf4..7052dea9de 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -2,6 +2,7 @@ mod add_transaction_views; mod add_utxo_account; mod addresses_table; mod initial_setup; +mod nullifier_map; mod received_notes_nullable_nf; mod sent_notes_to_internal; mod shardtree_support; @@ -30,6 +31,10 @@ pub(super) fn all_migrations( // add_transaction_views // / // v_transactions_net + // / + // received_notes_nullable_nf + // / \ + // shardtree_support nullifier_map vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -48,5 +53,6 @@ pub(super) fn all_migrations( Box::new(v_transactions_net::Migration), Box::new(received_notes_nullable_nf::Migration), Box::new(shardtree_support::Migration), + Box::new(nullifier_map::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/nullifier_map.rs b/zcash_client_sqlite/src/wallet/init/migrations/nullifier_map.rs new file mode 100644 index 0000000000..704b88e138 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/nullifier_map.rs @@ -0,0 +1,72 @@ +//! This migration adds a table for storing mappings from nullifiers to the transaction +//! they are revealed in. + +use std::collections::HashSet; + +use schemer_rusqlite::RusqliteMigration; +use tracing::debug; +use uuid::Uuid; + +use crate::wallet::init::WalletMigrationError; + +use super::received_notes_nullable_nf; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xe2d71ac5_6a44_4c6b_a9a0_6d0a79d355f1); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [received_notes_nullable_nf::MIGRATION_ID] + .into_iter() + .collect() + } + + fn description(&self) -> &'static str { + "Adds a lookup table for nullifiers we've observed on-chain that we haven't confirmed are not ours." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + // We don't enforce any foreign key constraint to the blocks table, to allow + // loading the nullifier map separately from block scanning. + debug!("Creating tables for nullifier map"); + transaction.execute_batch( + "CREATE TABLE tx_locator_map ( + block_height INTEGER NOT NULL, + tx_index INTEGER NOT NULL, + txid BLOB NOT NULL UNIQUE, + PRIMARY KEY (block_height, tx_index) + ); + CREATE TABLE nullifier_map ( + spend_pool INTEGER NOT NULL, + nf BLOB NOT NULL, + block_height INTEGER NOT NULL, + tx_index INTEGER NOT NULL, + CONSTRAINT tx_locator + FOREIGN KEY (block_height, tx_index) + REFERENCES tx_locator_map(block_height, tx_index) + ON DELETE CASCADE + ON UPDATE RESTRICT, + CONSTRAINT nf_uniq UNIQUE (spend_pool, nf) + );", + )?; + + Ok(()) + } + + fn down(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + transaction.execute_batch( + "DROP TABLE nullifier_map; + DROP TABLE tx_locator_map;", + )?; + Ok(()) + } +} diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 5108db19e2..335782307e 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -312,10 +312,11 @@ pub(crate) fn put_received_note( conn: &Connection, output: &T, tx_ref: i64, + spent_in: Option, ) -> Result { let mut stmt_upsert_received_note = conn.prepare_cached( "INSERT INTO sapling_received_notes - (tx, output_index, account, diversifier, value, rcm, memo, nf, is_change, commitment_tree_position) + (tx, output_index, account, diversifier, value, rcm, memo, nf, is_change, spent, commitment_tree_position) VALUES ( :tx, :output_index, @@ -326,6 +327,7 @@ pub(crate) fn put_received_note( :memo, :nf, :is_change, + :spent, :commitment_tree_position ) ON CONFLICT (tx, output_index) DO UPDATE @@ -336,6 +338,7 @@ pub(crate) fn put_received_note( nf = IFNULL(:nf, nf), memo = IFNULL(:memo, memo), is_change = IFNULL(:is_change, is_change), + spent = IFNULL(:spent, spent), commitment_tree_position = IFNULL(:commitment_tree_position, commitment_tree_position) RETURNING id_note", )?; @@ -354,6 +357,7 @@ pub(crate) fn put_received_note( ":nf": output.nullifier().map(|nf| nf.0.as_ref()), ":memo": memo_repr(output.memo()), ":is_change": output.is_change(), + ":spent": spent_in, ":commitment_tree_position": output.note_commitment_tree_position().map(u64::from), ]; From 97a3f818ec990e537f338eb6f8707f14328478e8 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 21 Jul 2023 20:59:24 +0000 Subject: [PATCH 0077/1122] zcash_client_sqlite: Only import recent frontiers into `ShardTree` We only need to load frontiers into the ShardTree that are close enough to the wallet's known chain tip to fill `PRUNING_DEPTH` checkpoints, so that ShardTree's witness generation will be able to correctly handle anchor depths. Loading frontiers further back than this doesn't add any useful nodes to the ShardTree (as we don't support rollbacks beyond `PRUNING_DEPTH`, and we won't be finding notes in earlier blocks), and hurts performance (as frontier importing has a significant Merkle tree hashing cost). Closes zcash/librustzcash#877. --- .../init/migrations/shardtree_support.rs | 57 ++++++++++++------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index f8c4a61af1..663a1c37d1 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -103,6 +103,8 @@ impl RusqliteMigration for Migration { );", )?; + let block_height_extrema = block_height_extrema(transaction)?; + let shard_store = SqliteShardStore::<_, sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( transaction, @@ -149,25 +151,40 @@ impl RusqliteMigration for Migration { stmt_update_block_sapling_tree_size .execute(params![block_end_tree.size(), block_height])?; - if let Some(nonempty_frontier) = block_end_tree.to_frontier().value() { - trace!( - height = block_height, - frontier = ?nonempty_frontier, - "Inserting frontier nodes", - ); - shard_tree - .insert_frontier_nodes( - nonempty_frontier.clone(), - Retention::Checkpoint { - id: BlockHeight::from(block_height), - is_marked: false, - }, - ) - .map_err(|e| match e { - ShardTreeError::Query(e) => ShardTreeError::Query(e), - ShardTreeError::Insert(e) => ShardTreeError::Insert(e), - ShardTreeError::Storage(_) => unreachable!(), - })? + // We only need to load frontiers into the ShardTree that are close enough + // to the wallet's known chain tip to fill `PRUNING_DEPTH` checkpoints, so + // that ShardTree's witness generation will be able to correctly handle + // anchor depths. Loading frontiers further back than this doesn't add any + // useful nodes to the ShardTree (as we don't support rollbacks beyond + // `PRUNING_DEPTH`, and we won't be finding notes in earlier blocks), and + // hurts performance (as frontier importing has a significant Merkle tree + // hashing cost). + if let Some((nonempty_frontier, (_, latest_height))) = block_end_tree + .to_frontier() + .value() + .zip(block_height_extrema) + { + let block_height = BlockHeight::from(block_height); + if block_height + PRUNING_DEPTH >= latest_height { + trace!( + height = u32::from(block_height), + frontier = ?nonempty_frontier, + "Inserting frontier nodes", + ); + shard_tree + .insert_frontier_nodes( + nonempty_frontier.clone(), + Retention::Checkpoint { + id: block_height, + is_marked: false, + }, + ) + .map_err(|e| match e { + ShardTreeError::Query(e) => ShardTreeError::Query(e), + ShardTreeError::Insert(e) => ShardTreeError::Insert(e), + ShardTreeError::Storage(_) => unreachable!(), + })? + } } } } @@ -241,7 +258,7 @@ impl RusqliteMigration for Migration { );", )?; - if let Some((start, end)) = block_height_extrema(transaction)? { + if let Some((start, end)) = block_height_extrema { // `ScanRange` uses an exclusive upper bound. let chain_end = end + 1; insert_queue_entries( From ed06a37b099609f8870e85b0c4511389e6063326 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 21 Jul 2023 21:01:03 +0000 Subject: [PATCH 0078/1122] zcash_client_sqlite: Mark existing blocks scanned in `shardtree` migration The `shardtree` migration is applied to a database state that was created via linear scanning, so we have complete wallet information for those blocks. --- .../src/wallet/init/migrations/shardtree_support.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 663a1c37d1..1abba5f423 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -265,7 +265,7 @@ impl RusqliteMigration for Migration { transaction, Some(ScanRange::from_parts( start..chain_end, - ScanPriority::Historic, + ScanPriority::Scanned, )) .iter(), )?; From b33330f9caf2c9920e9e72c4c8d6a2ef6c0f9c13 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 25 Jul 2023 02:14:18 +0000 Subject: [PATCH 0079/1122] zcash_client_backend: Accept owned keys in `scan_block_with_runner` This internal helper function can now take owned or borrowed keys. The public `scan_block` function is left unaltered. --- zcash_client_backend/CHANGELOG.md | 4 +++- zcash_client_backend/src/scanning.rs | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 1d77875dd6..92ab4ef4a7 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -23,7 +23,9 @@ and this library adheres to Rust's notion of - `testing::MockWalletDb::new` - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}`: - `zcash_client_backend::wallet::WalletSaplingOutput::note_commitment_tree_position` -- `zcash_client_backend::scanning::ScanError` +- `zcash_client_backend::scanning`: + - `ScanError` + - `impl ScanningKey for &K` ### Changed - MSRV is now 1.65.0. diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index 0f25571f99..ada2840819 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -66,6 +66,21 @@ pub trait ScanningKey { -> Self::Nf; } +impl ScanningKey for &K { + type Scope = K::Scope; + type SaplingNk = K::SaplingNk; + type SaplingKeys = K::SaplingKeys; + type Nf = K::Nf; + + fn to_sapling_keys(&self) -> Self::SaplingKeys { + (*self).to_sapling_keys() + } + + fn sapling_nf(key: &Self::SaplingNk, note: &sapling::Note, position: Position) -> Self::Nf { + K::sapling_nf(key, note, position) + } +} + impl ScanningKey for DiversifiableFullViewingKey { type Scope = Scope; type SaplingNk = sapling::NullifierDerivingKey; @@ -312,7 +327,7 @@ pub(crate) fn scan_block_with_runner< >( params: &P, block: CompactBlock, - vks: &[(&AccountId, &K)], + vks: &[(&AccountId, K)], nullifiers: &[(AccountId, sapling::Nullifier)], prior_block_metadata: Option<&BlockMetadata>, mut batch_runner: Option<&mut TaggedBatchRunner>, From 2d3d5e8fe6b8b43293868fd85b7c62cb18aae765 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 25 Jul 2023 02:21:15 +0000 Subject: [PATCH 0080/1122] zcash_client_backend: Precompute IVKs in `scan_cached_blocks` --- zcash_client_backend/CHANGELOG.md | 1 + zcash_client_backend/src/data_api/chain.rs | 13 +++++++++++-- zcash_client_backend/src/scanning.rs | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 92ab4ef4a7..f74aed8868 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -26,6 +26,7 @@ and this library adheres to Rust's notion of - `zcash_client_backend::scanning`: - `ScanError` - `impl ScanningKey for &K` + - `impl ScanningKey for (zip32::Scope, sapling::SaplingIvk, sapling::NullifierDerivingKey)` ### Changed - MSRV is now 1.65.0. diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 0f6f94545d..e6c5125178 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -155,7 +155,7 @@ use crate::{ data_api::{BlockMetadata, NullifierQuery, WalletWrite}, proto::compact_formats::CompactBlock, scan::BatchRunner, - scanning::{add_block_to_runner, check_continuity, scan_block_with_runner}, + scanning::{add_block_to_runner, check_continuity, scan_block_with_runner, ScanningKey}, }; pub mod error; @@ -254,6 +254,15 @@ where .iter() .filter_map(|(account, ufvk)| ufvk.sapling().map(move |k| (account, k))) .collect(); + // Precompute the IVKs instead of doing so per block. + let ivks = dfvks + .iter() + .flat_map(|(account, dfvk)| { + dfvk.to_sapling_keys() + .into_iter() + .map(|key| (*account, key)) + }) + .collect::>(); // Get the nullifiers for the unspent notes we are tracking let mut sapling_nullifiers = data_db @@ -337,7 +346,7 @@ where let scanned_block = scan_block_with_runner( params, block, - &dfvks, + &ivks, &sapling_nullifiers, prior_block_metadata.as_ref(), Some(&mut batch_runner), diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index ada2840819..f8ad561548 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -107,6 +107,21 @@ impl ScanningKey for DiversifiableFullViewingKey { } } +impl ScanningKey for (Scope, SaplingIvk, sapling::NullifierDerivingKey) { + type Scope = Scope; + type SaplingNk = sapling::NullifierDerivingKey; + type SaplingKeys = [(Self::Scope, SaplingIvk, Self::SaplingNk); 1]; + type Nf = sapling::Nullifier; + + fn to_sapling_keys(&self) -> Self::SaplingKeys { + [self.clone()] + } + + fn sapling_nf(key: &Self::SaplingNk, note: &sapling::Note, position: Position) -> Self::Nf { + note.nf(key, position.into()) + } +} + /// The [`ScanningKey`] implementation for [`SaplingIvk`]s. /// Nullifiers cannot be derived when scanning with these keys. /// From e773cd3ed460c2ced29fa1af059cf2fcf9cc1a71 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 25 Jul 2023 13:45:52 +0000 Subject: [PATCH 0081/1122] zcash_client_sqlite: Add index for foreign key of `nullifier_map` table This greatly improves the performance of `wallet::prune_nullifier_map` by speeding up `ON DELETE CASCADE`. --- .../src/wallet/init/migrations/nullifier_map.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/nullifier_map.rs b/zcash_client_sqlite/src/wallet/init/migrations/nullifier_map.rs index 704b88e138..06300e5875 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/nullifier_map.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/nullifier_map.rs @@ -56,7 +56,8 @@ impl RusqliteMigration for Migration { ON DELETE CASCADE ON UPDATE RESTRICT, CONSTRAINT nf_uniq UNIQUE (spend_pool, nf) - );", + ); + CREATE INDEX nf_map_locator_idx ON nullifier_map(block_height, tx_index);", )?; Ok(()) From 0f6970aefd151984196b33ec643a48ed0d193f32 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 25 Jul 2023 13:46:44 +0000 Subject: [PATCH 0082/1122] zcash_client_sqlite: Cache statement for `wallet::prune_nullifier_map` --- zcash_client_sqlite/src/wallet.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 23704cdd35..c7fb373c76 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1474,12 +1474,13 @@ pub(crate) fn prune_nullifier_map( conn: &rusqlite::Transaction<'_>, block_height: BlockHeight, ) -> Result<(), SqliteClientError> { - conn.execute( + let mut stmt_delete_locators = conn.prepare_cached( "DELETE FROM tx_locator_map WHERE block_height < :block_height", - named_params![":block_height": u32::from(block_height)], )?; + stmt_delete_locators.execute(named_params![":block_height": u32::from(block_height)])?; + Ok(()) } From 99ecdf3a894f03f4ebe4fecaf1273147bf8294ae Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 25 Jul 2023 13:57:10 +0000 Subject: [PATCH 0083/1122] zcash_client_sqlite: Build subtrees from new commitments in a threadpool The new `multicore` feature flag can be used to disable this behaviour. --- Cargo.toml | 4 ++-- zcash_client_sqlite/CHANGELOG.md | 3 +++ zcash_client_sqlite/Cargo.toml | 3 +++ zcash_client_sqlite/src/lib.rs | 31 ++++++++++++++++++++++++++++--- 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 983e50e242..61240bab2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,6 @@ panic = 'abort' codegen-units = 1 [patch.crates-io] -incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "bae25ad89c0c192bee625252d2d419bd56638e48" } -shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "bae25ad89c0c192bee625252d2d419bd56638e48" } +incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "2a667f500958d517c6b2ea608477140afd907fd8" } +shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "2a667f500958d517c6b2ea608477140afd907fd8" } orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 91f75a523d..1bde3251b0 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -9,6 +9,9 @@ and this library adheres to Rust's notion of ### Added - `zcash_client_sqlite::serialization` Serialization formats for data stored as SQLite BLOBs in the wallet database. +- A new default-enabled feature flag `multicore`. This allows users to disable + multicore support by setting `default_features = false` on their + `zcash_primitives`, `zcash_proofs`, and `zcash_client_sqlite` dependencies. ### Changed - MSRV is now 1.65.0. diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 6a951ff108..e0ce340038 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -48,6 +48,7 @@ uuid = "1.1" # Dependencies used internally: # (Breaking upgrades to these are usually backwards-compatible, but check MSRVs.) +maybe-rayon = {version = "0.1.0", default-features = false} [dev-dependencies] assert_matches = "1.5" @@ -63,6 +64,8 @@ zcash_primitives = { version = "0.12", path = "../zcash_primitives", features = zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } [features] +default = ["multicore"] +multicore = ["maybe-rayon/threads", "zcash_primitives/multicore"] mainnet = [] test-dependencies = [ "incrementalmerkletree/test-dependencies", diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 425dfe54e6..51b21dce57 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -33,6 +33,10 @@ #![deny(rustdoc::broken_intra_doc_links)] use either::Either; +use maybe_rayon::{ + prelude::{IndexedParallelIterator, ParallelIterator}, + slice::ParallelSliceMut, +}; use rusqlite::{self, Connection}; use secrecy::{ExposeSecret, SecretVec}; use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, io, ops::Range, path::Path}; @@ -474,7 +478,7 @@ impl WalletWrite for WalletDb })); last_scanned_height = Some(block.height()); - sapling_commitments.extend(block.into_sapling_commitments().into_iter()); + sapling_commitments.extend(block.into_sapling_commitments().into_iter().map(Some)); } // Prune the nullifier map of entries we no longer need. @@ -490,10 +494,31 @@ impl WalletWrite for WalletDb if let Some(((start_height, start_position), last_scanned_height)) = start_positions.zip(last_scanned_height) { + // Create subtrees from the note commitments in parallel. + const CHUNK_SIZE: usize = 1024; + let subtrees = sapling_commitments + .par_chunks_mut(CHUNK_SIZE) + .enumerate() + .filter_map(|(i, chunk)| { + let start = start_position + (i * CHUNK_SIZE) as u64; + let end = start + chunk.len() as u64; + + shardtree::LocatedTree::from_iter( + start..end, + SAPLING_SHARD_HEIGHT.into(), + chunk.iter_mut().map(|n| n.take().expect("always Some")), + ) + }) + .map(|res| (res.subtree, res.checkpoints)) + .collect::>(); + // Update the Sapling note commitment tree with all newly read note commitments - let mut sapling_commitments = sapling_commitments.into_iter(); + let mut subtrees = subtrees.into_iter(); wdb.with_sapling_tree_mut::<_, _, SqliteClientError>(move |sapling_tree| { - sapling_tree.batch_insert(start_position, &mut sapling_commitments)?; + for (tree, checkpoints) in &mut subtrees { + sapling_tree.insert_tree(tree, checkpoints)?; + } + Ok(()) })?; From facd4ccac56bb646917513c2d87f5e8d8b59750c Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 25 Jul 2023 20:57:49 +0000 Subject: [PATCH 0084/1122] Migrate to latest `shardtree` revision --- Cargo.toml | 4 ++-- zcash_client_backend/src/data_api.rs | 4 ++-- zcash_client_backend/src/data_api/error.rs | 3 ++- zcash_client_backend/src/data_api/wallet.rs | 2 +- zcash_client_sqlite/src/error.rs | 2 +- zcash_client_sqlite/src/lib.rs | 2 +- zcash_client_sqlite/src/wallet/commitment_tree.rs | 5 +++-- zcash_client_sqlite/src/wallet/init.rs | 2 +- .../src/wallet/init/migrations/shardtree_support.rs | 2 +- 9 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 61240bab2a..c673a73a40 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,6 @@ panic = 'abort' codegen-units = 1 [patch.crates-io] -incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "2a667f500958d517c6b2ea608477140afd907fd8" } -shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "2a667f500958d517c6b2ea608477140afd907fd8" } +incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } +shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 8b27ca3df8..606c09635f 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -7,7 +7,7 @@ use std::num::NonZeroU32; use incrementalmerkletree::Retention; use secrecy::SecretVec; -use shardtree::{ShardStore, ShardTree, ShardTreeError}; +use shardtree::{error::ShardTreeError, store::ShardStore, ShardTree}; use zcash_primitives::{ block::BlockHash, consensus::BlockHeight, @@ -584,7 +584,7 @@ pub trait WalletCommitmentTrees { pub mod testing { use incrementalmerkletree::Address; use secrecy::{ExposeSecret, SecretVec}; - use shardtree::{memory::MemoryShardStore, ShardTree, ShardTreeError}; + use shardtree::{error::ShardTreeError, store::memory::MemoryShardStore, ShardTree}; use std::{collections::HashMap, convert::Infallible}; use zcash_primitives::{ diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index db4ffb9842..31a9b39c99 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -1,8 +1,9 @@ //! Types for wallet error handling. -use shardtree::ShardTreeError; use std::error; use std::fmt::{self, Debug, Display}; + +use shardtree::error::ShardTreeError; use zcash_primitives::{ transaction::{ builder, diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 60ef900eea..630ff80daf 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1,7 +1,7 @@ use std::fmt::Debug; use std::{convert::Infallible, num::NonZeroU32}; -use shardtree::{ShardStore, ShardTree, ShardTreeError}; +use shardtree::{error::ShardTreeError, store::ShardStore, ShardTree}; use zcash_primitives::{ consensus::{self, BlockHeight, NetworkUpgrade}, memo::MemoBytes, diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 9058359d46..741973b176 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -5,7 +5,7 @@ use std::error; use std::fmt; use std::io; -use shardtree::ShardTreeError; +use shardtree::error::ShardTreeError; use zcash_client_backend::encoding::{Bech32DecodeError, TransparentCodecError}; use zcash_primitives::{consensus::BlockHeight, zip32::AccountId}; diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 51b21dce57..aede6ff26b 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -42,7 +42,7 @@ use secrecy::{ExposeSecret, SecretVec}; use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, io, ops::Range, path::Path}; use incrementalmerkletree::Position; -use shardtree::{ShardTree, ShardTreeError}; +use shardtree::{error::ShardTreeError, ShardTree}; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight}, diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index be5c38b42b..da29626b18 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -10,8 +10,9 @@ use zcash_client_backend::data_api::chain::CommitmentTreeRoot; use incrementalmerkletree::{Address, Hashable, Level, Position, Retention}; use shardtree::{ - Checkpoint, LocatedPrunableTree, LocatedTree, PrunableTree, RetentionFlags, ShardStore, - ShardTreeError, TreeState, + error::ShardTreeError, + store::{Checkpoint, ShardStore, TreeState}, + LocatedPrunableTree, LocatedTree, PrunableTree, RetentionFlags, }; use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer}; diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index c5d9ab7639..a589e97881 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -8,7 +8,7 @@ use rusqlite::{self, types::ToSql}; use schemer::{Migrator, MigratorError}; use schemer_rusqlite::RusqliteAdapter; use secrecy::SecretVec; -use shardtree::{ShardTree, ShardTreeError}; +use shardtree::{error::ShardTreeError, ShardTree}; use uuid::Uuid; use zcash_primitives::{ diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 1abba5f423..204d5e0b69 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -8,7 +8,7 @@ use incrementalmerkletree::Retention; use rusqlite::{self, named_params, params}; use schemer; use schemer_rusqlite::RusqliteMigration; -use shardtree::{caching::CachingShardStore, ShardTree, ShardTreeError}; +use shardtree::{error::ShardTreeError, store::caching::CachingShardStore, ShardTree}; use tracing::{debug, trace}; use uuid::Uuid; From e640f82ff65099264f2a80e7dff19fdcdb2b4f76 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 2 Aug 2023 10:22:54 -0600 Subject: [PATCH 0085/1122] Pin `time` dependency to avoid forced MSRV bump. Co-authored-by: Jack Grigg --- zcash_client_backend/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 7d0e463823..3f47a951cd 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -89,6 +89,8 @@ tempfile = "3.5.0" zcash_proofs = { version = "0.12", path = "../zcash_proofs", default-features = false } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } +time = ">=0.3.22, <0.3.24" # time 0.3.24 has MSRV 1.67 + [features] lightwalletd-tonic = ["tonic"] transparent-inputs = ["hdwallet", "zcash_primitives/transparent-inputs"] From e718e769899f4e39253304c4f879bc79b9a5bd95 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 1 Aug 2023 09:10:59 -0600 Subject: [PATCH 0086/1122] zcash_client_sqlite: Add a test that demonstrates the expected behavior of `get_memo` for empty-memo situations. --- zcash_client_sqlite/src/wallet/sapling.rs | 180 ++++++++++++++++++++-- 1 file changed, 168 insertions(+), 12 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 335782307e..4e52f7f1d1 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -371,7 +371,7 @@ pub(crate) fn put_received_note( #[cfg(test)] #[allow(deprecated)] pub(crate) mod tests { - use std::num::NonZeroU32; + use std::{convert::Infallible, num::NonZeroU32}; use rusqlite::Connection; use secrecy::Secret; @@ -383,8 +383,13 @@ pub(crate) mod tests { block::BlockHash, consensus::{BlockHeight, BranchId}, legacy::TransparentAddress, + memo::Memo, sapling::{note_encryption::try_sapling_output_recovery, prover::TxProver}, - transaction::{components::Amount, fees::zip317::FeeRule as Zip317FeeRule, Transaction}, + transaction::{ + components::Amount, + fees::{fixed::FeeRule as FixedFeeRule, zip317::FeeRule as Zip317FeeRule}, + Transaction, + }, zip32::{sapling::ExtendedSpendingKey, Scope}, }; @@ -394,13 +399,17 @@ pub(crate) mod tests { self, chain::scan_cached_blocks, error::Error, - wallet::{create_spend_to_address, input_selection::GreedyInputSelector, spend}, + wallet::{ + create_proposed_transaction, create_spend_to_address, + input_selection::GreedyInputSelector, propose_transfer, spend, + }, WalletRead, WalletWrite, }, - fees::{zip317, DustOutputPolicy}, + decrypt_transaction, + fees::{fixed, zip317, DustOutputPolicy}, keys::UnifiedSpendingKey, wallet::OvkPolicy, - zip321::{Payment, TransactionRequest}, + zip321::{self, Payment, TransactionRequest}, }; use crate::{ @@ -413,21 +422,17 @@ pub(crate) mod tests { get_balance, get_balance_at, init::{init_blocks_table, init_wallet_db}, }, - AccountId, BlockDb, WalletDb, + AccountId, BlockDb, NoteId, WalletDb, }; #[cfg(feature = "transparent-inputs")] use { zcash_client_backend::{ - data_api::wallet::shield_transparent_funds, fees::fixed, - wallet::WalletTransparentOutput, + data_api::wallet::shield_transparent_funds, wallet::WalletTransparentOutput, }, zcash_primitives::{ memo::MemoBytes, - transaction::{ - components::{amount::NonNegativeAmount, OutPoint, TxOut}, - fees::fixed::FeeRule as FixedFeeRule, - }, + transaction::components::{amount::NonNegativeAmount, OutPoint, TxOut}, }, }; @@ -440,6 +445,157 @@ pub(crate) mod tests { } } + #[test] + fn send_proposed_transfer() { + let cache_file = NamedTempFile::new().unwrap(); + let db_cache = BlockDb(Connection::open(cache_file.path()).unwrap()); + init_cache_database(&db_cache).unwrap(); + + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + init_wallet_db(&mut db_data, None).unwrap(); + + // Add an account to the wallet + let seed = Secret::new([0u8; 32].to_vec()); + let (account, usk) = db_data.create_account(&seed).unwrap(); + let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); + + // Add funds to the wallet in a single note + let value = Amount::from_u64(60000).unwrap(); + let (cb, _) = fake_compact_block( + sapling_activation_height(), + BlockHash([0; 32]), + &dfvk, + AddressType::DefaultExternal, + value, + 0, + ); + insert_into_cache(&db_cache, &cb); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + sapling_activation_height(), + 1, + ) + .unwrap(); + + // Verified balance matches total balance + let (_, anchor_height) = db_data + .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) + .unwrap() + .unwrap(); + assert_eq!( + get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + value + ); + assert_eq!( + get_balance_at(&db_data.conn, AccountId::from(0), anchor_height).unwrap(), + value + ); + + let to_extsk = ExtendedSpendingKey::master(&[]); + let to: RecipientAddress = to_extsk.default_address().1.into(); + let request = zip321::TransactionRequest::new(vec![Payment { + recipient_address: to, + amount: Amount::from_u64(10000).unwrap(), + memo: None, // this should result in the creation of an empty memo + label: None, + message: None, + other_params: vec![], + }]) + .unwrap(); + + let fee_rule = FixedFeeRule::standard(); + let change_strategy = fixed::SingleOutputChangeStrategy::new(fee_rule); + let input_selector = + &GreedyInputSelector::new(change_strategy, DustOutputPolicy::default()); + let proposal_result = propose_transfer::<_, _, _, Infallible>( + &mut db_data, + &tests::network(), + account, + input_selector, + request, + NonZeroU32::new(1).unwrap(), + ); + assert_matches!(proposal_result, Ok(_)); + + let change_memo = "Test change memo".parse::().unwrap(); + let create_proposed_result = create_proposed_transaction::<_, _, Infallible, _>( + &mut db_data, + &tests::network(), + test_prover(), + &usk, + OvkPolicy::Sender, + proposal_result.unwrap(), + NonZeroU32::new(1).unwrap(), + Some(change_memo.clone().into()), + ); + assert_matches!(create_proposed_result, Ok(_)); + + let sent_tx_id = create_proposed_result.unwrap(); + + // Verify that the sent transaction was stored and that we can decrypt the memos + let tx = db_data + .get_transaction(sent_tx_id) + .expect("Created transaction was stored."); + let ufvks = [(account, usk.to_unified_full_viewing_key())] + .into_iter() + .collect(); + let decrypted_outputs = decrypt_transaction( + &tests::network(), + sapling_activation_height() + 1, + &tx, + &ufvks, + ); + + let mut found_tx_change_memo = false; + let mut found_tx_empty_memo = false; + for output in decrypted_outputs { + if output.memo == change_memo.clone().into() { + found_tx_change_memo = true + } + if output.memo == Memo::Empty.into() { + found_tx_empty_memo = true + } + } + assert!(found_tx_change_memo); + assert!(found_tx_empty_memo); + + // Verify that the stored sent notes match what we're expecting + let mut stmt_sent_notes = db_data + .conn + .prepare("SELECT id_note FROM sent_notes WHERE tx = ?") + .unwrap(); + + let sent_note_ids = stmt_sent_notes + .query(rusqlite::params![sent_tx_id]) + .unwrap() + .mapped(|row| row.get::<_, i64>(0).map(NoteId::SentNoteId)) + .collect::, _>>() + .unwrap(); + + assert_eq!(sent_note_ids.len(), 2); + + // The sent memo should be the empty memo for both the sent output and change + let mut found_sent_change_memo = false; + let mut found_sent_empty_memo = false; + for sent_note_id in sent_note_ids { + let memo = db_data.get_memo(sent_note_id).expect("Note id is valid"); + if memo.as_ref() == Some(&change_memo) { + found_sent_change_memo = true + } + if memo == Some(Memo::Empty) { + found_sent_empty_memo = true + } + } + assert!(found_sent_change_memo); + assert!(found_sent_empty_memo); + + // Check that querying for a nonexistent sent note returns an error + assert_matches!(db_data.get_memo(NoteId::SentNoteId(12345)), Err(_)); + } + #[test] fn create_to_address_fails_on_incorrect_usk() { let data_file = NamedTempFile::new().unwrap(); From 40ecaaf758b199e660c19cdefe2ca39d21f698a2 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 1 Aug 2023 11:06:00 -0600 Subject: [PATCH 0087/1122] zcash_client_backend: Fix error where change memos were not being included in sent transactions. --- zcash_client_backend/src/data_api/wallet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 630ff80daf..1bda4a8c8d 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -588,7 +588,7 @@ where internal_ovk(), dfvk.change_address().1, *amount, - MemoBytes::empty(), + change_memo.clone().unwrap_or_else(MemoBytes::empty), )?; sapling_output_meta.push(( Recipient::InternalAccount( From f0b2fc3421afd9bb11bef03a2e09b94b57632dbb Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 1 Aug 2023 12:00:10 -0600 Subject: [PATCH 0088/1122] zcash_client_sqlite: Store the empty memo as a single 0xf6 byte. --- zcash_client_sqlite/src/wallet.rs | 12 +++++++++--- zcash_client_sqlite/src/wallet/sapling.rs | 19 +++++++++++++------ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index c7fb373c76..71591dc2a0 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -129,8 +129,14 @@ pub(crate) fn pool_code(pool_type: PoolType) -> i64 { } pub(crate) fn memo_repr(memo: Option<&MemoBytes>) -> Option<&[u8]> { - memo.filter(|m| *m != &MemoBytes::empty()) - .map(|m| m.as_slice()) + memo.map(|m| { + if m == &MemoBytes::empty() { + // we store the empty memo as a single 0xf6 byte + &[0xf6] + } else { + m.as_slice() + } + }) } pub(crate) fn get_max_account_id( @@ -1262,7 +1268,7 @@ pub(crate) fn insert_sent_output( ":to_address": &to_address, ":to_account": &to_account, ":value": &i64::from(output.value()), - ":memo": output.memo().filter(|m| *m != &MemoBytes::empty()).map(|m| m.as_slice()), + ":memo": memo_repr(output.memo()) ]; stmt_insert_sent_output.execute(sql_args)?; diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 4e52f7f1d1..280cbe53aa 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -581,12 +581,19 @@ pub(crate) mod tests { let mut found_sent_change_memo = false; let mut found_sent_empty_memo = false; for sent_note_id in sent_note_ids { - let memo = db_data.get_memo(sent_note_id).expect("Note id is valid"); - if memo.as_ref() == Some(&change_memo) { - found_sent_change_memo = true - } - if memo == Some(Memo::Empty) { - found_sent_empty_memo = true + match db_data + .get_memo(sent_note_id) + .expect("Note id is valid") + .as_ref() + { + Some(m) if m == &change_memo => { + found_sent_change_memo = true; + } + Some(m) if m == &Memo::Empty => { + found_sent_empty_memo = true; + } + Some(other) => panic!("Unexpected memo value: {:?}", other), + None => panic!("Memo should not be stored as NULL"), } } assert!(found_sent_change_memo); From a5a8562ddde1a5d7b7a2dbe9497fb7adfabe65c4 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 1 Aug 2023 12:00:58 -0600 Subject: [PATCH 0089/1122] zcash_client_backend, zcash_client_sqlite: Ensure consistency between sent & stored memos. --- zcash_client_backend/src/data_api/wallet.rs | 32 +-- zcash_client_sqlite/src/lib.rs | 2 +- zcash_client_sqlite/src/wallet.rs | 4 +- zcash_client_sqlite/src/wallet/init.rs | 10 +- .../src/wallet/init/migrations.rs | 30 +-- .../migrations/sapling_memo_consistency.rs | 218 ++++++++++++++++++ 6 files changed, 262 insertions(+), 34 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 1bda4a8c8d..2d09231e8d 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -545,30 +545,29 @@ where for payment in proposal.transaction_request().payments() { match &payment.recipient_address { RecipientAddress::Unified(ua) => { + let memo = payment + .memo + .as_ref() + .map_or_else(MemoBytes::empty, |m| m.clone()); builder.add_sapling_output( external_ovk, *ua.sapling().expect("TODO: Add Orchard support to builder"), payment.amount, - payment.memo.clone().unwrap_or_else(MemoBytes::empty), + memo.clone(), )?; sapling_output_meta.push(( Recipient::Unified(ua.clone(), PoolType::Shielded(ShieldedProtocol::Sapling)), payment.amount, - payment.memo.clone(), + Some(memo), )); } RecipientAddress::Shielded(addr) => { - builder.add_sapling_output( - external_ovk, - *addr, - payment.amount, - payment.memo.clone().unwrap_or_else(MemoBytes::empty), - )?; - sapling_output_meta.push(( - Recipient::Sapling(*addr), - payment.amount, - payment.memo.clone(), - )); + let memo = payment + .memo + .as_ref() + .map_or_else(MemoBytes::empty, |m| m.clone()); + builder.add_sapling_output(external_ovk, *addr, payment.amount, memo.clone())?; + sapling_output_meta.push((Recipient::Sapling(*addr), payment.amount, Some(memo))); } RecipientAddress::Transparent(to) => { if payment.memo.is_some() { @@ -584,11 +583,14 @@ where for change_value in proposal.balance().proposed_change() { match change_value { ChangeValue::Sapling(amount) => { + let memo = change_memo + .as_ref() + .map_or_else(MemoBytes::empty, |m| m.clone()); builder.add_sapling_output( internal_ovk(), dfvk.change_address().1, *amount, - change_memo.clone().unwrap_or_else(MemoBytes::empty), + memo.clone(), )?; sapling_output_meta.push(( Recipient::InternalAccount( @@ -596,7 +598,7 @@ where PoolType::Shielded(ShieldedProtocol::Sapling), ), *amount, - change_memo.clone(), + Some(memo), )) } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index aede6ff26b..3c97d60a9a 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -240,7 +240,7 @@ impl, P: consensus::Parameters> WalletRead for W } fn get_transaction(&self, id_tx: i64) -> Result { - wallet::get_transaction(self.conn.borrow(), &self.params, id_tx) + wallet::get_transaction(self.conn.borrow(), &self.params, id_tx).map(|(_, tx)| tx) } fn get_sapling_nullifiers( diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 71591dc2a0..14d5e28831 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -503,7 +503,7 @@ pub(crate) fn get_transaction( conn: &rusqlite::Connection, params: &P, id_tx: i64, -) -> Result { +) -> Result<(BlockHeight, Transaction), SqliteClientError> { let (tx_bytes, block_height, expiry_height): ( Vec<_>, Option, @@ -536,6 +536,7 @@ pub(crate) fn get_transaction( block_height.or_else(|| expiry_height.filter(|h| h > &BlockHeight::from(0))) { Transaction::read(&tx_bytes[..], BranchId::for_height(params, height)) + .map(|t| (height, t)) .map_err(SqliteClientError::from) } else { let tx_data = Transaction::read(&tx_bytes[..], BranchId::Sprout) @@ -555,6 +556,7 @@ pub(crate) fn get_transaction( tx_data.orchard_bundle().cloned(), ) .freeze() + .map(|t| (expiry_height, t)) .map_err(SqliteClientError::from) } else { Err(SqliteClientError::CorruptedData( diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index a589e97881..8ba478c883 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -574,8 +574,9 @@ mod tests { ELSE 1 END AS received_count, CASE - WHEN sapling_received_notes.memo IS NULL THEN 0 - ELSE 1 + WHEN (sapling_received_notes.memo IS NULL OR sapling_received_notes.memo = X'F6') + THEN 0 + ELSE 1 END AS memo_present FROM sapling_received_notes UNION @@ -606,8 +607,9 @@ mod tests { COUNT(DISTINCT sent_notes.id_note) as sent_notes, SUM( CASE - WHEN sent_notes.memo IS NULL THEN 0 - ELSE 1 + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6') + THEN 0 + ELSE 1 END ) AS memo_count FROM sent_notes diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 7052dea9de..25d29c68ca 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -4,6 +4,7 @@ mod addresses_table; mod initial_setup; mod nullifier_map; mod received_notes_nullable_nf; +mod sapling_memo_consistency; mod sent_notes_to_internal; mod shardtree_support; mod ufvk_support; @@ -22,19 +23,19 @@ pub(super) fn all_migrations( ) -> Vec>> { // initial_setup // / \ - // utxos_table ufvk_support ---------- - // \ \ \ - // \ addresses_table sent_notes_to_internal - // \ / / - // add_utxo_account / - // \ / - // add_transaction_views - // / - // v_transactions_net - // / - // received_notes_nullable_nf - // / \ - // shardtree_support nullifier_map + // utxos_table ufvk_support --------- + // \ | \ + // \ addresses_table sent_notes_to_internal + // \ / / + // add_utxo_account / + // \ / + // add_transaction_views + // | + // v_transactions_net + // | + // received_notes_nullable_nf + // / | \ + // shardtree_support nullifier_map sapling_memo_consistency vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -54,5 +55,8 @@ pub(super) fn all_migrations( Box::new(received_notes_nullable_nf::Migration), Box::new(shardtree_support::Migration), Box::new(nullifier_map::Migration), + Box::new(sapling_memo_consistency::Migration { + params: params.clone(), + }), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs b/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs new file mode 100644 index 0000000000..e491365f51 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs @@ -0,0 +1,218 @@ +//! This migration reads decrypted transaction data and updates the `sent_notes` table to ensure +//! that memo entries are consistent with the decrypted transaction's outputs. The empty memo is +//! now consistently represented as a single `0xf6` byte. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use rusqlite::named_params; +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; +use zcash_client_backend::{decrypt_transaction, keys::UnifiedFullViewingKey}; +use zcash_primitives::{consensus, zip32::AccountId}; + +use crate::{ + error::SqliteClientError, + wallet::{get_transaction, init::WalletMigrationError, memo_repr}, +}; + +use super::received_notes_nullable_nf; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x7029b904_6557_4aa1_9da5_6904b65d2ba5); + +pub(super) struct Migration

{ + pub(super) params: P, +} + +impl

schemer::Migration for Migration

{ + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [received_notes_nullable_nf::MIGRATION_ID] + .into_iter() + .collect() + } + + fn description(&self) -> &'static str { + "" + } +} + +impl RusqliteMigration for Migration

{ + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + let mut stmt_raw_tx = transaction.prepare( + "SELECT DISTINCT sent_notes.tx, sent_notes.output_index, accounts.account, accounts.ufvk + FROM sent_notes + JOIN accounts ON sent_notes.from_account = accounts.account + JOIN transactions ON transactions.id_tx = sent_notes.tx + WHERE transactions.raw IS NOT NULL" + )?; + + let mut rows = stmt_raw_tx.query([])?; + + let mut tx_sent_notes: BTreeMap> = + BTreeMap::new(); + while let Some(row) = rows.next()? { + let id_tx: i64 = row.get(0)?; + let output_index: u32 = row.get(1)?; + let account: u32 = row.get(2)?; + let ufvk_str: String = row.get(3)?; + let ufvk = UnifiedFullViewingKey::decode(&self.params, &ufvk_str).map_err(|e| { + WalletMigrationError::CorruptedData(format!( + "Could not decode unified full viewing key for account {}: {:?}", + account, e + )) + })?; + + tx_sent_notes + .entry(id_tx) + .and_modify(|v| v.push((output_index, AccountId::from(account), ufvk.clone()))) + .or_insert(vec![(output_index, AccountId::from(account), ufvk)]); + } + + for (id_tx, sapling_outputs) in tx_sent_notes { + let (block_height, tx) = + get_transaction(transaction, &self.params, id_tx).map_err(|err| match err { + SqliteClientError::CorruptedData(msg) => { + WalletMigrationError::CorruptedData(msg) + } + SqliteClientError::DbError(err) => WalletMigrationError::DbError(err), + other => WalletMigrationError::CorruptedData(format!( + "An error was encountered decoding transaction data: {:?}", + other + )), + })?; + + let ufvks: HashMap = sapling_outputs + .iter() + .map(|(_, account, ufvk)| (*account, ufvk.clone())) + .collect(); + + let decrypted_outputs = decrypt_transaction(&self.params, block_height, &tx, &ufvks); + let mut stmt_update_sent_memo = transaction.prepare( + "UPDATE sent_notes + SET memo = :memo + WHERE tx = :id_tx + AND output_index = :output_index", + )?; + + for d_out in decrypted_outputs { + if sapling_outputs + .iter() + .any(|(idx, _, _)| &(u32::try_from(d_out.index).unwrap()) == idx) + { + stmt_update_sent_memo.execute(named_params![ + ":id_tx": id_tx, + ":output_index": d_out.index, + ":memo": memo_repr(Some(&d_out.memo)) + ])?; + } + } + } + + // Update the `v_transactions` view to avoid counting the empty memo as a memo + transaction.execute_batch( + "DROP VIEW v_transactions; + CREATE VIEW v_transactions AS + WITH + notes AS ( + SELECT sapling_received_notes.account AS account_id, + sapling_received_notes.tx AS id_tx, + 2 AS pool, + sapling_received_notes.value AS value, + CASE + WHEN sapling_received_notes.is_change THEN 1 + ELSE 0 + END AS is_change, + CASE + WHEN sapling_received_notes.is_change THEN 0 + ELSE 1 + END AS received_count, + CASE + WHEN (sapling_received_notes.memo IS NULL OR sapling_received_notes.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present + FROM sapling_received_notes + UNION + SELECT utxos.received_by_account AS account_id, + transactions.id_tx AS id_tx, + 0 AS pool, + utxos.value_zat AS value, + 0 AS is_change, + 1 AS received_count, + 0 AS memo_present + FROM utxos + JOIN transactions + ON transactions.txid = utxos.prevout_txid + UNION + SELECT sapling_received_notes.account AS account_id, + sapling_received_notes.spent AS id_tx, + 2 AS pool, + -sapling_received_notes.value AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM sapling_received_notes + WHERE sapling_received_notes.spent IS NOT NULL + ), + sent_note_counts AS ( + SELECT sent_notes.from_account AS account_id, + sent_notes.tx AS id_tx, + COUNT(DISTINCT sent_notes.id_note) as sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6') + THEN 0 + ELSE 1 + END + ) AS memo_count + FROM sent_notes + LEFT JOIN sapling_received_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + WHERE sapling_received_notes.is_change IS NULL + OR sapling_received_notes.is_change = 0 + GROUP BY account_id, id_tx + ), + blocks_max_height AS ( + SELECT MAX(blocks.height) as max_height FROM blocks + ) + SELECT notes.account_id AS account_id, + transactions.id_tx AS id_tx, + transactions.block AS mined_height, + transactions.tx_index AS tx_index, + transactions.txid AS txid, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.is_change) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( + blocks.height IS NULL + AND transactions.expiry_height <= blocks_max_height.max_height + ) AS expired_unmined + FROM transactions + JOIN notes ON notes.id_tx = transactions.id_tx + JOIN blocks_max_height + LEFT JOIN blocks ON blocks.height = transactions.block + LEFT JOIN sent_note_counts + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.id_tx = notes.id_tx + GROUP BY notes.account_id, transactions.id_tx", + )?; + + Ok(()) + } + + fn down(&self, _: &rusqlite::Transaction) -> Result<(), Self::Error> { + panic!("Reversing this migration is not supported."); + } +} From 90412ebdc230cbc113945a42ec86124116d09dca Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 4 Aug 2023 14:11:24 -0600 Subject: [PATCH 0090/1122] Fix SentTransactionOutput documentation. --- zcash_client_backend/src/data_api.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 606c09635f..fc09fee04f 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -451,12 +451,14 @@ impl SentTransactionOutput { pub fn value(&self) -> Amount { self.value } - /// Returns the memo that was attached to the output, if any. + /// Returns the memo that was attached to the output, if any. This will only be `None` + /// for transparent outputs. pub fn memo(&self) -> Option<&MemoBytes> { self.memo.as_ref() } - /// Returns t decrypted note, if the sent output belongs to this wallet + /// Returns the account to which change (or wallet-internal value in the case of a shielding + /// transaction) was sent, along with the change note. pub fn sapling_change_to(&self) -> Option<&(AccountId, sapling::Note)> { self.sapling_change_to.as_ref() } From cb0d16cfa97c2532d97d58dfc890213f96cddf45 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 4 Aug 2023 14:35:58 -0600 Subject: [PATCH 0091/1122] Fix clippy complaint. --- .../src/wallet/init/migrations/sapling_memo_consistency.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs b/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs index e491365f51..08caeb1039 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs @@ -70,7 +70,7 @@ impl RusqliteMigration for Migration

{ tx_sent_notes .entry(id_tx) .and_modify(|v| v.push((output_index, AccountId::from(account), ufvk.clone()))) - .or_insert(vec![(output_index, AccountId::from(account), ufvk)]); + .or_insert_with(|| vec![(output_index, AccountId::from(account), ufvk)]); } for (id_tx, sapling_outputs) in tx_sent_notes { From 0ee45e40c4d1e0cf4a2c1398b42d84ad9664351d Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 3 Aug 2023 11:19:40 -0600 Subject: [PATCH 0092/1122] zcash_client_sqlite: Replace Either-based definition of `wallet::commitment_tree::Error` with a bespoke error type. --- zcash_client_sqlite/Cargo.toml | 1 - zcash_client_sqlite/src/error.rs | 9 +- zcash_client_sqlite/src/lib.rs | 23 ++- .../src/wallet/commitment_tree.rs | 157 ++++++++++-------- zcash_client_sqlite/src/wallet/init.rs | 12 +- 5 files changed, 112 insertions(+), 90 deletions(-) diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index e0ce340038..0d409717ad 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -32,7 +32,6 @@ tracing = "0.1" # - Serialization byteorder = "1" prost = "0.11" -either = "1.8" group = "0.13" jubjub = "0.10" diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 741973b176..dec1aa7091 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -1,14 +1,13 @@ //! Error types for problems that may arise when reading or storing wallet data to SQLite. -use either::Either; use std::error; use std::fmt; -use std::io; use shardtree::error::ShardTreeError; use zcash_client_backend::encoding::{Bech32DecodeError, TransparentCodecError}; use zcash_primitives::{consensus::BlockHeight, zip32::AccountId}; +use crate::wallet::commitment_tree; use crate::PRUNING_DEPTH; #[cfg(feature = "transparent-inputs")] @@ -85,7 +84,7 @@ pub enum SqliteClientError { /// An error occurred in inserting data into or accessing data from one of the wallet's note /// commitment trees. - CommitmentTree(ShardTreeError>), + CommitmentTree(ShardTreeError), } impl error::Error for SqliteClientError { @@ -176,8 +175,8 @@ impl From for SqliteClientError { } } -impl From>> for SqliteClientError { - fn from(e: ShardTreeError>) -> Self { +impl From> for SqliteClientError { + fn from(e: ShardTreeError) -> Self { SqliteClientError::CommitmentTree(e) } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index aede6ff26b..4a30a3a4b7 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -32,7 +32,6 @@ // Catch documentation errors caused by code changes. #![deny(rustdoc::broken_intra_doc_links)] -use either::Either; use maybe_rayon::{ prelude::{IndexedParallelIterator, ParallelIterator}, slice::ParallelSliceMut, @@ -86,7 +85,7 @@ pub mod error; pub mod serialization; pub mod wallet; -use wallet::commitment_tree::put_shard_roots; +use wallet::commitment_tree::{self, put_shard_roots}; /// The maximum number of blocks the wallet is allowed to rewind. This is /// consistent with the bound in zcashd, and allows block data deeper than @@ -726,7 +725,7 @@ impl WalletWrite for WalletDb } impl WalletCommitmentTrees for WalletDb { - type Error = Either; + type Error = commitment_tree::Error; type SaplingShardStore<'a> = SqliteShardStore<&'a rusqlite::Transaction<'a>, sapling::Node, SAPLING_SHARD_HEIGHT>; @@ -739,21 +738,21 @@ impl WalletCommitmentTrees for WalletDb, ) -> Result, - E: From>>, + E: From>, { let tx = self .conn .transaction() - .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?; let shard_store = SqliteShardStore::from_connection(&tx, SAPLING_TABLES_PREFIX) - .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?; let result = { let mut shardtree = ShardTree::new(shard_store, PRUNING_DEPTH.try_into().unwrap()); callback(&mut shardtree)? }; tx.commit() - .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?; Ok(result) } @@ -765,7 +764,7 @@ impl WalletCommitmentTrees for WalletDb( &tx, SAPLING_TABLES_PREFIX, @@ -773,13 +772,13 @@ impl WalletCommitmentTrees for WalletDb WalletCommitmentTrees for WalletDb, P> { - type Error = Either; + type Error = commitment_tree::Error; type SaplingShardStore<'a> = SqliteShardStore<&'a rusqlite::Transaction<'a>, sapling::Node, SAPLING_SHARD_HEIGHT>; @@ -792,11 +791,11 @@ impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb, ) -> Result, - E: From>>, + E: From>, { let mut shardtree = ShardTree::new( SqliteShardStore::from_connection(self.conn.0, SAPLING_TABLES_PREFIX) - .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?, + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?, PRUNING_DEPTH.try_into().unwrap(), ); let result = callback(&mut shardtree)?; diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index da29626b18..12554cc6f6 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -1,7 +1,7 @@ -use either::Either; use rusqlite::{self, named_params, OptionalExtension}; use std::{ collections::BTreeSet, + error, fmt, io::{self, Cursor}, marker::PhantomData, sync::Arc, @@ -19,6 +19,33 @@ use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer}; use crate::serialization::{read_shard, write_shard}; +/// Errors that can appear in SQLite-back [`ShardStore`] implementation operations. +#[derive(Debug)] +pub enum Error { + /// Errors in deserializing stored shard data + Serialization(io::Error), + /// Errors encountered querying stored shard data + Query(rusqlite::Error), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self { + Error::Serialization(err) => write!(f, "Commitment tree serializtion error: {}", err), + Error::Query(err) => write!(f, "Commitment tree query or update error: {}", err), + } + } +} + +impl error::Error for Error { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + match &self { + Error::Serialization(e) => Some(e), + Error::Query(e) => Some(e), + } + } +} + pub struct SqliteShardStore { pub(crate) conn: C, table_prefix: &'static str, @@ -45,7 +72,7 @@ impl<'conn, 'a: 'conn, H: HashSer, const SHARD_HEIGHT: u8> ShardStore { type H = H; type CheckpointId = BlockHeight; - type Error = Either; + type Error = Error; fn get_shard( &self, @@ -147,7 +174,7 @@ impl ShardStore { type H = H; type CheckpointId = BlockHeight; - type Error = Either; + type Error = Error; fn get_shard( &self, @@ -161,9 +188,9 @@ impl ShardStore } fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { - let tx = self.conn.transaction().map_err(Either::Right)?; + let tx = self.conn.transaction().map_err(Error::Query)?; put_shard(&tx, self.table_prefix, subtree)?; - tx.commit().map_err(Either::Right)?; + tx.commit().map_err(Error::Query)?; Ok(()) } @@ -196,9 +223,9 @@ impl ShardStore checkpoint_id: Self::CheckpointId, checkpoint: Checkpoint, ) -> Result<(), Self::Error> { - let tx = self.conn.transaction().map_err(Either::Right)?; + let tx = self.conn.transaction().map_err(Error::Query)?; add_checkpoint(&tx, self.table_prefix, checkpoint_id, checkpoint)?; - tx.commit().map_err(Either::Right) + tx.commit().map_err(Error::Query) } fn checkpoint_count(&self) -> Result { @@ -223,9 +250,9 @@ impl ShardStore where F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, { - let tx = self.conn.transaction().map_err(Either::Right)?; + let tx = self.conn.transaction().map_err(Error::Query)?; with_checkpoints(&tx, self.table_prefix, limit, callback)?; - tx.commit().map_err(Either::Right) + tx.commit().map_err(Error::Query) } fn update_checkpoint_with( @@ -236,30 +263,28 @@ impl ShardStore where F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, { - let tx = self.conn.transaction().map_err(Either::Right)?; + let tx = self.conn.transaction().map_err(Error::Query)?; let result = update_checkpoint_with(&tx, self.table_prefix, *checkpoint_id, update)?; - tx.commit().map_err(Either::Right)?; + tx.commit().map_err(Error::Query)?; Ok(result) } fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { - let tx = self.conn.transaction().map_err(Either::Right)?; + let tx = self.conn.transaction().map_err(Error::Query)?; remove_checkpoint(&tx, self.table_prefix, *checkpoint_id)?; - tx.commit().map_err(Either::Right) + tx.commit().map_err(Error::Query) } fn truncate_checkpoints( &mut self, checkpoint_id: &Self::CheckpointId, ) -> Result<(), Self::Error> { - let tx = self.conn.transaction().map_err(Either::Right)?; + let tx = self.conn.transaction().map_err(Error::Query)?; truncate_checkpoints(&tx, self.table_prefix, *checkpoint_id)?; - tx.commit().map_err(Either::Right) + tx.commit().map_err(Error::Query) } } -type Error = Either; - pub(crate) fn get_shard( conn: &rusqlite::Connection, table_prefix: &'static str, @@ -276,12 +301,12 @@ pub(crate) fn get_shard( |row| Ok((row.get::<_, Vec>(0)?, row.get::<_, Option>>(1)?)), ) .optional() - .map_err(Either::Right)? + .map_err(Error::Query)? .map(|(shard_data, root_hash)| { - let shard_tree = read_shard(&mut Cursor::new(shard_data)).map_err(Either::Left)?; + let shard_tree = read_shard(&mut Cursor::new(shard_data)).map_err(Error::Serialization)?; let located_tree = LocatedPrunableTree::from_parts(shard_root_addr, shard_tree); if let Some(root_hash_data) = root_hash { - let root_hash = H::read(Cursor::new(root_hash_data)).map_err(Either::Left)?; + let root_hash = H::read(Cursor::new(root_hash_data)).map_err(Error::Serialization)?; Ok(located_tree.reannotate_root(Some(Arc::new(root_hash)))) } else { Ok(located_tree) @@ -311,10 +336,10 @@ pub(crate) fn last_shard( }, ) .optional() - .map_err(Either::Right)? + .map_err(Error::Query)? .map(|(shard_index, shard_data)| { let shard_root = Address::from_parts(shard_root_level, shard_index); - let shard_tree = read_shard(&mut Cursor::new(shard_data)).map_err(Either::Left)?; + let shard_tree = read_shard(&mut Cursor::new(shard_data)).map_err(Error::Serialization)?; Ok(LocatedPrunableTree::from_parts(shard_root, shard_tree)) }) .transpose() @@ -336,10 +361,10 @@ pub(crate) fn put_shard( }) }) .transpose() - .map_err(Either::Left)?; + .map_err(Error::Serialization)?; let mut subtree_data = vec![]; - write_shard(&mut subtree_data, subtree.root()).map_err(Either::Left)?; + write_shard(&mut subtree_data, subtree.root()).map_err(Error::Serialization)?; let mut stmt_put_shard = conn .prepare_cached(&format!( @@ -350,7 +375,7 @@ pub(crate) fn put_shard( shard_data = :shard_data", table_prefix )) - .map_err(Either::Right)?; + .map_err(Error::Query)?; stmt_put_shard .execute(named_params![ @@ -358,7 +383,7 @@ pub(crate) fn put_shard( ":root_hash": subtree_root_hash, ":shard_data": subtree_data ]) - .map_err(Either::Right)?; + .map_err(Error::Query)?; Ok(()) } @@ -373,14 +398,14 @@ pub(crate) fn get_shard_roots( "SELECT shard_index FROM {}_tree_shards ORDER BY shard_index", table_prefix )) - .map_err(Either::Right)?; - let mut rows = stmt.query([]).map_err(Either::Right)?; + .map_err(Error::Query)?; + let mut rows = stmt.query([]).map_err(Error::Query)?; let mut res = vec![]; - while let Some(row) = rows.next().map_err(Either::Right)? { + while let Some(row) = rows.next().map_err(Error::Query)? { res.push(Address::from_parts( shard_root_level, - row.get(0).map_err(Either::Right)?, + row.get(0).map_err(Error::Query)?, )); } Ok(res) @@ -398,7 +423,7 @@ pub(crate) fn truncate( ), [from.index()], ) - .map_err(Either::Right) + .map_err(Error::Query) .map(|_| ()) } @@ -412,10 +437,10 @@ pub(crate) fn get_cap( |row| row.get::<_, Vec>(0), ) .optional() - .map_err(Either::Right)? + .map_err(Error::Query)? .map_or_else( || Ok(PrunableTree::empty()), - |cap_data| read_shard(&mut Cursor::new(cap_data)).map_err(Either::Left), + |cap_data| read_shard(&mut Cursor::new(cap_data)).map_err(Error::Serialization), ) } @@ -432,11 +457,11 @@ pub(crate) fn put_cap( SET cap_data = :cap_data", table_prefix )) - .map_err(Either::Right)?; + .map_err(Error::Query)?; let mut cap_data = vec![]; - write_shard(&mut cap_data, &cap).map_err(Either::Left)?; - stmt.execute([cap_data]).map_err(Either::Right)?; + write_shard(&mut cap_data, &cap).map_err(Error::Serialization)?; + stmt.execute([cap_data]).map_err(Error::Query)?; Ok(()) } @@ -456,7 +481,7 @@ pub(crate) fn min_checkpoint_id( .map(|opt| opt.map(BlockHeight::from)) }, ) - .map_err(Either::Right) + .map_err(Error::Query) } pub(crate) fn max_checkpoint_id( @@ -474,7 +499,7 @@ pub(crate) fn max_checkpoint_id( .map(|opt| opt.map(BlockHeight::from)) }, ) - .map_err(Either::Right) + .map_err(Error::Query) } pub(crate) fn add_checkpoint( @@ -489,14 +514,14 @@ pub(crate) fn add_checkpoint( VALUES (:checkpoint_id, :position)", table_prefix )) - .map_err(Either::Right)?; + .map_err(Error::Query)?; stmt_insert_checkpoint .execute(named_params![ ":checkpoint_id": u32::from(checkpoint_id), ":position": checkpoint.position().map(u64::from) ]) - .map_err(Either::Right)?; + .map_err(Error::Query)?; let mut stmt_insert_mark_removed = conn .prepare_cached(&format!( @@ -504,7 +529,7 @@ pub(crate) fn add_checkpoint( VALUES (:checkpoint_id, :position)", table_prefix )) - .map_err(Either::Right)?; + .map_err(Error::Query)?; for pos in checkpoint.marks_removed() { stmt_insert_mark_removed @@ -512,7 +537,7 @@ pub(crate) fn add_checkpoint( ":checkpoint_id": u32::from(checkpoint_id), ":position": u64::from(*pos) ]) - .map_err(Either::Right)?; + .map_err(Error::Query)?; } Ok(()) @@ -527,7 +552,7 @@ pub(crate) fn checkpoint_count( [], |row| row.get::<_, usize>(0), ) - .map_err(Either::Right) + .map_err(Error::Query) } pub(crate) fn get_checkpoint( @@ -550,7 +575,7 @@ pub(crate) fn get_checkpoint( }, ) .optional() - .map_err(Either::Right)?; + .map_err(Error::Query)?; checkpoint_position .map(|pos_opt| { @@ -561,15 +586,15 @@ pub(crate) fn get_checkpoint( WHERE checkpoint_id = ?", table_prefix )) - .map_err(Either::Right)?; + .map_err(Error::Query)?; let mark_removed_rows = stmt .query([u32::from(checkpoint_id)]) - .map_err(Either::Right)?; + .map_err(Error::Query)?; let marks_removed = mark_removed_rows .mapped(|row| row.get::<_, u64>(0).map(Position::from)) .collect::, _>>() - .map_err(Either::Right)?; + .map_err(Error::Query)?; Ok(Checkpoint::from_parts( pos_opt.map_or(TreeState::Empty, TreeState::AtPosition), @@ -609,7 +634,7 @@ pub(crate) fn get_checkpoint_at_depth( }, ) .optional() - .map_err(Either::Right)?; + .map_err(Error::Query)?; checkpoint_parts .map(|(checkpoint_id, pos_opt)| { @@ -620,15 +645,15 @@ pub(crate) fn get_checkpoint_at_depth( WHERE checkpoint_id = ?", table_prefix )) - .map_err(Either::Right)?; + .map_err(Error::Query)?; let mark_removed_rows = stmt .query([u32::from(checkpoint_id)]) - .map_err(Either::Right)?; + .map_err(Error::Query)?; let marks_removed = mark_removed_rows .mapped(|row| row.get::<_, u64>(0).map(Position::from)) .collect::, _>>() - .map_err(Either::Right)?; + .map_err(Error::Query)?; Ok(( checkpoint_id, @@ -658,7 +683,7 @@ where LIMIT :limit", table_prefix )) - .map_err(Either::Right)?; + .map_err(Error::Query)?; let mut stmt_get_checkpoint_marks_removed = conn .prepare_cached(&format!( @@ -667,27 +692,27 @@ where WHERE checkpoint_id = :checkpoint_id", table_prefix )) - .map_err(Either::Right)?; + .map_err(Error::Query)?; let mut rows = stmt_get_checkpoints .query(named_params![":limit": limit]) - .map_err(Either::Right)?; + .map_err(Error::Query)?; - while let Some(row) = rows.next().map_err(Either::Right)? { - let checkpoint_id = row.get::<_, u32>(0).map_err(Either::Right)?; + while let Some(row) = rows.next().map_err(Error::Query)? { + let checkpoint_id = row.get::<_, u32>(0).map_err(Error::Query)?; let tree_state = row .get::<_, Option>(1) .map(|opt| opt.map_or_else(|| TreeState::Empty, |p| TreeState::AtPosition(p.into()))) - .map_err(Either::Right)?; + .map_err(Error::Query)?; let mark_removed_rows = stmt_get_checkpoint_marks_removed .query(named_params![":checkpoint_id": checkpoint_id]) - .map_err(Either::Right)?; + .map_err(Error::Query)?; let marks_removed = mark_removed_rows .mapped(|row| row.get::<_, u64>(0).map(Position::from)) .collect::, _>>() - .map_err(Either::Right)?; + .map_err(Error::Query)?; callback( &BlockHeight::from(checkpoint_id), @@ -730,11 +755,11 @@ pub(crate) fn remove_checkpoint( WHERE checkpoint_id = :checkpoint_id", table_prefix )) - .map_err(Either::Right)?; + .map_err(Error::Query)?; stmt_delete_checkpoint .execute(named_params![":checkpoint_id": u32::from(checkpoint_id),]) - .map_err(Either::Right)?; + .map_err(Error::Query)?; Ok(()) } @@ -753,7 +778,7 @@ pub(crate) fn truncate_checkpoints( ), [u32::from(checkpoint_id)], ) - .map_err(Either::Right)?; + .map_err(Error::Query)?; Ok(()) } @@ -844,18 +869,18 @@ pub(crate) fn put_shard_roots< SET subtree_end_height = :subtree_end_height, root_hash = :root_hash", table_prefix )) - .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; + .map_err(|e| ShardTreeError::Storage(Error::Query(e)))?; // The `shard_data` value will only be used in the case that no tree already exists. let mut shard_data: Vec = vec![]; let tree = PrunableTree::leaf((root.root_hash().clone(), RetentionFlags::EPHEMERAL)); write_shard(&mut shard_data, &tree) - .map_err(|e| ShardTreeError::Storage(Either::Left(e)))?; + .map_err(|e| ShardTreeError::Storage(Error::Serialization(e)))?; let mut root_hash_data: Vec = vec![]; root.root_hash() .write(&mut root_hash_data) - .map_err(|e| ShardTreeError::Storage(Either::Left(e)))?; + .map_err(|e| ShardTreeError::Storage(Error::Serialization(e)))?; stmt.execute(named_params![ ":shard_index": start_index + i, @@ -863,7 +888,7 @@ pub(crate) fn put_shard_roots< ":root_hash": root_hash_data, ":shard_data": shard_data, ]) - .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; + .map_err(|e| ShardTreeError::Storage(Error::Query(e)))?; } Ok(()) diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index a589e97881..6388a40d9e 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -1,7 +1,7 @@ //! Functions for initializing the various databases. -use either::Either; + use incrementalmerkletree::Retention; -use std::{collections::HashMap, fmt, io}; +use std::{collections::HashMap, fmt}; use tracing::debug; use rusqlite::{self, types::ToSql}; @@ -24,7 +24,7 @@ use zcash_client_backend::{data_api::SAPLING_SHARD_HEIGHT, keys::UnifiedFullView use crate::{error::SqliteClientError, wallet, WalletDb, PRUNING_DEPTH, SAPLING_TABLES_PREFIX}; -use super::commitment_tree::SqliteShardStore; +use super::commitment_tree::{self, SqliteShardStore}; mod migrations; @@ -43,7 +43,7 @@ pub enum WalletMigrationError { BalanceError(BalanceError), /// Wrapper for commitment tree invariant violations - CommitmentTree(ShardTreeError>), + CommitmentTree(ShardTreeError), } impl From for WalletMigrationError { @@ -58,8 +58,8 @@ impl From for WalletMigrationError { } } -impl From>> for WalletMigrationError { - fn from(e: ShardTreeError>) -> Self { +impl From> for WalletMigrationError { + fn from(e: ShardTreeError) -> Self { WalletMigrationError::CommitmentTree(e) } } From 027b3c9af85faf21d6f18bfcc5f3c1fe16ed3f19 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 3 Aug 2023 11:19:40 -0600 Subject: [PATCH 0093/1122] zcash_client_sqlite: Ensure that re-adding the same checkpoint information does not cause a database conflict. The `add_checkpoint` method is intended to be idempotent. In the case that we add a checkpoint at an already-checkpointed block height, we should only raise an error in the case that the note commitment tree position or the set of notes spent in the checkpointed block has changed. --- zcash_client_sqlite/CHANGELOG.md | 2 + zcash_client_sqlite/src/lib.rs | 4 +- zcash_client_sqlite/src/wallet.rs | 2 +- .../src/wallet/commitment_tree.rs | 170 +++++++++++++----- zcash_client_sqlite/src/wallet/init.rs | 3 +- .../init/migrations/shardtree_support.rs | 3 +- 6 files changed, 133 insertions(+), 51 deletions(-) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 1bde3251b0..8700b10425 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -12,6 +12,8 @@ and this library adheres to Rust's notion of - A new default-enabled feature flag `multicore`. This allows users to disable multicore support by setting `default_features = false` on their `zcash_primitives`, `zcash_proofs`, and `zcash_client_sqlite` dependencies. +- `zcash_client_sqlite::wallet::commitment_tree` A new module containing a + sqlite-backed implementation of `shardtree::store::ShardStore`. ### Changed - MSRV is now 1.65.0. diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 4a30a3a4b7..c37a46b4cc 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -38,7 +38,7 @@ use maybe_rayon::{ }; use rusqlite::{self, Connection}; use secrecy::{ExposeSecret, SecretVec}; -use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, io, ops::Range, path::Path}; +use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, ops::Range, path::Path}; use incrementalmerkletree::Position; use shardtree::{error::ShardTreeError, ShardTree}; @@ -76,8 +76,8 @@ use crate::{error::SqliteClientError, wallet::commitment_tree::SqliteShardStore} #[cfg(feature = "unstable")] use { crate::chain::{fsblockdb_with_blocks, BlockMeta}, - std::fs, std::path::PathBuf, + std::{fs, io}, }; pub mod chain; diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index c7fb373c76..b96ce6fbda 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -111,7 +111,7 @@ use { }, }; -pub(crate) mod commitment_tree; +pub mod commitment_tree; pub mod init; pub(crate) mod sapling; pub(crate) mod scanning; diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 12554cc6f6..bce22ca7bc 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -26,6 +26,15 @@ pub enum Error { Serialization(io::Error), /// Errors encountered querying stored shard data Query(rusqlite::Error), + /// Raised when the caller attempts to add a checkpoint at a block height where a checkpoint + /// already exists, but the tree state being checkpointed or the marks removed at that + /// checkpoint conflict with the existing tree state. + CheckpointConflict { + checkpoint_id: BlockHeight, + checkpoint: Checkpoint, + extant_tree_state: TreeState, + extant_marks_removed: Option>, + }, } impl fmt::Display for Error { @@ -33,6 +42,18 @@ impl fmt::Display for Error { match &self { Error::Serialization(err) => write!(f, "Commitment tree serializtion error: {}", err), Error::Query(err) => write!(f, "Commitment tree query or update error: {}", err), + Error::CheckpointConflict { + checkpoint_id, + checkpoint, + extant_tree_state, + extant_marks_removed, + } => { + write!( + f, + "Conflict at checkpoint id {}, tried to insert {:?}, which is incompatible with existing state ({:?}, {:?})", + checkpoint_id, checkpoint, extant_tree_state, extant_marks_removed + ) + } } } } @@ -42,6 +63,7 @@ impl error::Error for Error { match &self { Error::Serialization(e) => Some(e), Error::Query(e) => Some(e), + Error::CheckpointConflict { .. } => None, } } } @@ -508,39 +530,89 @@ pub(crate) fn add_checkpoint( checkpoint_id: BlockHeight, checkpoint: Checkpoint, ) -> Result<(), Error> { - let mut stmt_insert_checkpoint = conn - .prepare_cached(&format!( - "INSERT INTO {}_tree_checkpoints (checkpoint_id, position) - VALUES (:checkpoint_id, :position)", - table_prefix - )) + let extant_tree_state = conn + .query_row( + &format!( + "SELECT position FROM {}_tree_checkpoints WHERE checkpoint_id = :checkpoint_id", + table_prefix + ), + named_params![":checkpoint_id": u32::from(checkpoint_id),], + |row| { + row.get::<_, Option>(0).map(|opt| { + opt.map_or_else( + || TreeState::Empty, + |pos| TreeState::AtPosition(Position::from(pos)), + ) + }) + }, + ) + .optional() .map_err(Error::Query)?; - stmt_insert_checkpoint - .execute(named_params![ - ":checkpoint_id": u32::from(checkpoint_id), - ":position": checkpoint.position().map(u64::from) - ]) - .map_err(Error::Query)?; + match extant_tree_state { + Some(current) => { + if current != checkpoint.tree_state() { + // If the checkpoint position for a given checkpoint identifier has changed, we treat + // this as an error because the wallet should have detected a chain reorg and truncated + // the tree. + Err(Error::CheckpointConflict { + checkpoint_id, + checkpoint, + extant_tree_state: current, + extant_marks_removed: None, + }) + } else { + // if the existing spends are the same, we can skip the insert; if the + // existing spends have changed, this is also a conflict. + let marks_removed = get_marks_removed(conn, table_prefix, checkpoint_id)?; + if &marks_removed == checkpoint.marks_removed() { + Ok(()) + } else { + Err(Error::CheckpointConflict { + checkpoint_id, + checkpoint, + extant_tree_state: current, + extant_marks_removed: Some(marks_removed), + }) + } + } + } + None => { + let mut stmt_insert_checkpoint = conn + .prepare_cached(&format!( + "INSERT INTO {}_tree_checkpoints (checkpoint_id, position) + VALUES (:checkpoint_id, :position)", + table_prefix + )) + .map_err(Error::Query)?; - let mut stmt_insert_mark_removed = conn - .prepare_cached(&format!( - "INSERT INTO {}_tree_checkpoint_marks_removed (checkpoint_id, mark_removed_position) - VALUES (:checkpoint_id, :position)", - table_prefix - )) - .map_err(Error::Query)?; + stmt_insert_checkpoint + .execute(named_params![ + ":checkpoint_id": u32::from(checkpoint_id), + ":position": checkpoint.position().map(u64::from) + ]) + .map_err(Error::Query)?; - for pos in checkpoint.marks_removed() { - stmt_insert_mark_removed - .execute(named_params![ - ":checkpoint_id": u32::from(checkpoint_id), - ":position": u64::from(*pos) - ]) - .map_err(Error::Query)?; - } + let mut stmt_insert_mark_removed = conn + .prepare_cached(&format!( + "INSERT INTO {}_tree_checkpoint_marks_removed (checkpoint_id, mark_removed_position) + VALUES (:checkpoint_id, :position)", + table_prefix + )) + .map_err(Error::Query)?; - Ok(()) + for pos in checkpoint.marks_removed() { + stmt_insert_mark_removed + .execute(named_params![ + ":checkpoint_id": u32::from(checkpoint_id), + ":position": u64::from(*pos) + ]) + .map_err(Error::Query)?; + } + + Ok(()) + } + } } pub(crate) fn checkpoint_count( @@ -555,6 +627,29 @@ pub(crate) fn checkpoint_count( .map_err(Error::Query) } +fn get_marks_removed( + conn: &rusqlite::Connection, + table_prefix: &'static str, + checkpoint_id: BlockHeight, +) -> Result, Error> { + let mut stmt = conn + .prepare_cached(&format!( + "SELECT mark_removed_position + FROM {}_tree_checkpoint_marks_removed + WHERE checkpoint_id = ?", + table_prefix + )) + .map_err(Error::Query)?; + let mark_removed_rows = stmt + .query([u32::from(checkpoint_id)]) + .map_err(Error::Query)?; + + mark_removed_rows + .mapped(|row| row.get::<_, u64>(0).map(Position::from)) + .collect::, _>>() + .map_err(Error::Query) +} + pub(crate) fn get_checkpoint( conn: &rusqlite::Connection, table_prefix: &'static str, @@ -579,26 +674,9 @@ pub(crate) fn get_checkpoint( checkpoint_position .map(|pos_opt| { - let mut stmt = conn - .prepare_cached(&format!( - "SELECT mark_removed_position - FROM {}_tree_checkpoint_marks_removed - WHERE checkpoint_id = ?", - table_prefix - )) - .map_err(Error::Query)?; - let mark_removed_rows = stmt - .query([u32::from(checkpoint_id)]) - .map_err(Error::Query)?; - - let marks_removed = mark_removed_rows - .mapped(|row| row.get::<_, u64>(0).map(Position::from)) - .collect::, _>>() - .map_err(Error::Query)?; - Ok(Checkpoint::from_parts( pos_opt.map_or(TreeState::Empty, TreeState::AtPosition), - marks_removed, + get_marks_removed(conn, table_prefix, checkpoint_id)?, )) }) .transpose() diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 6388a40d9e..1225614d2e 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -454,7 +454,8 @@ mod tests { checkpoint_id INTEGER NOT NULL, mark_removed_position INTEGER NOT NULL, FOREIGN KEY (checkpoint_id) REFERENCES sapling_tree_checkpoints(checkpoint_id) - ON DELETE CASCADE + ON DELETE CASCADE, + CONSTRAINT spend_position_unique UNIQUE (checkpoint_id, mark_removed_position) )", "CREATE TABLE sapling_tree_checkpoints ( checkpoint_id INTEGER PRIMARY KEY, diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 204d5e0b69..64e1c42fc8 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -99,7 +99,8 @@ impl RusqliteMigration for Migration { checkpoint_id INTEGER NOT NULL, mark_removed_position INTEGER NOT NULL, FOREIGN KEY (checkpoint_id) REFERENCES sapling_tree_checkpoints(checkpoint_id) - ON DELETE CASCADE + ON DELETE CASCADE, + CONSTRAINT spend_position_unique UNIQUE (checkpoint_id, mark_removed_position) );", )?; From 522f1e4da0d1e9a63777881ec534f56834ea6f4f Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 7 Aug 2023 07:45:15 -0600 Subject: [PATCH 0094/1122] Apply suggestions from code review Co-authored-by: str4d --- zcash_client_sqlite/src/wallet.rs | 5 ++ .../migrations/sapling_memo_consistency.rs | 61 ++++++++----------- zcash_client_sqlite/src/wallet/sapling.rs | 4 +- 3 files changed, 34 insertions(+), 36 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 14d5e28831..e49901b789 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -499,6 +499,11 @@ pub(crate) fn get_received_memo( } /// Looks up a transaction by its internal database identifier. +/// +/// Returns the decoded transaction, along with the block height that was used in its decoding. +/// This is either the block height at which the transaction was mined, or the expiry height if the +/// wallet created the transaction but the transaction has not yet been mined from the perspective +/// of the wallet. pub(crate) fn get_transaction( conn: &rusqlite::Connection, params: &P, diff --git a/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs b/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs index 08caeb1039..4cfc983827 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs @@ -1,6 +1,6 @@ -//! This migration reads decrypted transaction data and updates the `sent_notes` table to ensure -//! that memo entries are consistent with the decrypted transaction's outputs. The empty memo is -//! now consistently represented as a single `0xf6` byte. +//! This migration reads the wallet's raw transaction data and updates the `sent_notes` table to +//! ensure that memo entries are consistent with the decrypted transaction's outputs. The empty +//! memo is now consistently represented as a single `0xf6` byte. use std::collections::{BTreeMap, HashMap, HashSet}; @@ -35,7 +35,9 @@ impl

schemer::Migration for Migration

{ } fn description(&self) -> &'static str { - "" + "This migration reads the wallet's raw transaction data and updates the `sent_notes` table to + ensure that memo entries are consistent with the decrypted transaction's outputs. The empty + memo is now consistently represented as a single `0xf6` byte." } } @@ -44,22 +46,21 @@ impl RusqliteMigration for Migration

{ fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { let mut stmt_raw_tx = transaction.prepare( - "SELECT DISTINCT sent_notes.tx, sent_notes.output_index, accounts.account, accounts.ufvk + "SELECT DISTINCT sent_notes.tx, accounts.account, accounts.ufvk FROM sent_notes JOIN accounts ON sent_notes.from_account = accounts.account JOIN transactions ON transactions.id_tx = sent_notes.tx - WHERE transactions.raw IS NOT NULL" + WHERE transactions.raw IS NOT NULL", )?; let mut rows = stmt_raw_tx.query([])?; - let mut tx_sent_notes: BTreeMap> = + let mut tx_sent_notes: BTreeMap> = BTreeMap::new(); while let Some(row) = rows.next()? { let id_tx: i64 = row.get(0)?; - let output_index: u32 = row.get(1)?; - let account: u32 = row.get(2)?; - let ufvk_str: String = row.get(3)?; + let account: u32 = row.get(1)?; + let ufvk_str: String = row.get(2)?; let ufvk = UnifiedFullViewingKey::decode(&self.params, &ufvk_str).map_err(|e| { WalletMigrationError::CorruptedData(format!( "Could not decode unified full viewing key for account {}: {:?}", @@ -69,11 +70,18 @@ impl RusqliteMigration for Migration

{ tx_sent_notes .entry(id_tx) - .and_modify(|v| v.push((output_index, AccountId::from(account), ufvk.clone()))) - .or_insert_with(|| vec![(output_index, AccountId::from(account), ufvk)]); + .or_default() + .insert(AccountId::from(account), ufvk); } - for (id_tx, sapling_outputs) in tx_sent_notes { + let mut stmt_update_sent_memo = transaction.prepare( + "UPDATE sent_notes + SET memo = :memo + WHERE tx = :id_tx + AND output_index = :output_index", + )?; + + for (id_tx, ufvks) in tx_sent_notes { let (block_height, tx) = get_transaction(transaction, &self.params, id_tx).map_err(|err| match err { SqliteClientError::CorruptedData(msg) => { @@ -86,30 +94,13 @@ impl RusqliteMigration for Migration

{ )), })?; - let ufvks: HashMap = sapling_outputs - .iter() - .map(|(_, account, ufvk)| (*account, ufvk.clone())) - .collect(); - let decrypted_outputs = decrypt_transaction(&self.params, block_height, &tx, &ufvks); - let mut stmt_update_sent_memo = transaction.prepare( - "UPDATE sent_notes - SET memo = :memo - WHERE tx = :id_tx - AND output_index = :output_index", - )?; - for d_out in decrypted_outputs { - if sapling_outputs - .iter() - .any(|(idx, _, _)| &(u32::try_from(d_out.index).unwrap()) == idx) - { - stmt_update_sent_memo.execute(named_params![ - ":id_tx": id_tx, - ":output_index": d_out.index, - ":memo": memo_repr(Some(&d_out.memo)) - ])?; - } + stmt_update_sent_memo.execute(named_params![ + ":id_tx": id_tx, + ":output_index": d_out.index, + ":memo": memo_repr(Some(&d_out.memo)) + ])?; } } diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 280cbe53aa..88cf3b2098 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -548,6 +548,7 @@ pub(crate) mod tests { &tx, &ufvks, ); + assert_eq!(decrypted_outputs.len(), 2); let mut found_tx_change_memo = false; let mut found_tx_empty_memo = false; @@ -577,7 +578,8 @@ pub(crate) mod tests { assert_eq!(sent_note_ids.len(), 2); - // The sent memo should be the empty memo for both the sent output and change + // The sent memo should be the empty memo for the sent output, and the + // change output's memo should be as specified. let mut found_sent_change_memo = false; let mut found_sent_empty_memo = false; for sent_note_id in sent_note_ids { From d3b7dffa3c582a48dbe633dbcde0b8e8cd8b9d0b Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 2 Aug 2023 13:45:49 -0600 Subject: [PATCH 0095/1122] zcash_client_backend: Restrict use of backend-specific note identifiers. In general, it is preferable to use globally relevant identifiers where possible. This PR removes the `WalletRead::TxRef` associated type in favor of using `TxId` directly for the transaction identifier, and restricts the use of the `NoteRef` type to those scenarios where the result of one query is intended to be used directly as the input to another query. Closes #834 --- zcash_client_backend/CHANGELOG.md | 23 +++-- zcash_client_backend/src/data_api.rs | 85 +++++++++++-------- zcash_client_backend/src/data_api/wallet.rs | 16 ++-- zcash_client_sqlite/CHANGELOG.md | 4 + zcash_client_sqlite/src/lib.rs | 68 +++++---------- zcash_client_sqlite/src/wallet.rs | 66 ++++++++------ zcash_client_sqlite/src/wallet/init.rs | 9 +- .../migrations/sapling_memo_consistency.rs | 19 +++-- zcash_client_sqlite/src/wallet/sapling.rs | 75 ++++++++-------- 9 files changed, 196 insertions(+), 169 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index f74aed8868..2b1602eb6d 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -12,6 +12,7 @@ and this library adheres to Rust's notion of - `impl Debug` for `zcash_client_backend::{data_api::wallet::input_selection::Proposal, wallet::ReceivedSaplingNote}` - `zcash_client_backend::data_api`: - `BlockMetadata` + - `NoteId` - `NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` - `ScannedBlock` - `ShieldedProtocol` @@ -33,9 +34,14 @@ and this library adheres to Rust's notion of - Bumped dependencies to `hdwallet 0.4`, `zcash_primitives 0.12`, `zcash_note_encryption 0.4`, `incrementalmerkletree 0.4`, `orchard 0.5`, `bs58 0.5` - `zcash_client_backend::data_api`: - - `WalletRead::get_memo` now returns `Result, Self::Error>` - instead of `Result` in order to make representable - wallet states where the full note plaintext is not available. + - `WalletRead::TxRef` has been removed in favor of consistently using `TxId` instead. + - `WalletRead::get_transaction` now takes a `TxId` as its argument. + - `WalletWrite::{store_decrypted_tx, store_sent_tx}` now return `Result<(), Self::Error>` + as the `WalletRead::TxRef` associated type has been removed. Use `TxId` instead. + - `WalletRead::get_memo` now takes a `NoteId` as its argument instead of `Self::NoteRef` + and returns `Result, Self::Error>` instead of `Result` in order to make representable wallet states where the full + note plaintext is not available. - `WalletRead::get_nullifiers` has been renamed to `WalletRead::get_sapling_nullifiers` and its signature has changed; it now subsumes the removed `WalletRead::get_all_nullifiers`. - `WalletRead::get_target_and_anchor_heights` now takes its argument as a `NonZeroU32` @@ -45,15 +51,15 @@ and this library adheres to Rust's notion of - `chain::BlockSource::with_blocks` now takes its limit as an `Option` instead of `Option`. - A new `CommitmentTree` variant has been added to `data_api::error::Error` - - `data_api::wallet::{create_spend_to_address, create_proposed_transaction, + - `wallet::{create_spend_to_address, create_proposed_transaction, shield_transparent_funds}` all now require that `WalletCommitmentTrees` be implemented for the type passed to them for the `wallet_db` parameter. - - `data_api::wallet::create_proposed_transaction` now takes an additional + - `wallet::create_proposed_transaction` now takes an additional `min_confirmations` argument. - - `data_api::wallet::{spend, create_spend_to_address, shield_transparent_funds, + - `wallet::{spend, create_spend_to_address, shield_transparent_funds, propose_transfer, propose_shielding, create_proposed_transaction}` now take their respective `min_confirmations` arguments as `NonZeroU32` - - `data_api::wallet::input_selection::InputSelector::{propose_transaction, propose_shielding}` + - `wallet::input_selection::InputSelector::{propose_transaction, propose_shielding}` now take their respective `min_confirmations` arguments as `NonZeroU32` - A new `Scan` variant has been added to `data_api::chain::error::Error`. - A new `SyncRequired` variant has been added to `data_api::wallet::input_selection::InputSelectorError`. @@ -64,6 +70,9 @@ and this library adheres to Rust's notion of - Arguments to `WalletSaplingOutput::from_parts` have changed. - `zcash_client_backend::data_api::wallet::input_selection::InputSelector`: - Arguments to `{propose_transaction, propose_shielding}` have changed. +- `zcash_client_backend::data_api::wallet::{create_spend_to_address, spend, + create_proposed_transaction, shield_transparent_funds}` now return the `TxId` + for the newly created transaction instead an internal database identifier. - `zcash_client_backend::wallet::ReceivedSaplingNote::note_commitment_tree_position` has replaced the `witness` field in the same struct. - `zcash_client_backend::welding_rig` has been renamed to `zcash_client_backend::scanning` diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index fc09fee04f..2e0562257e 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -38,6 +38,8 @@ pub mod wallet; pub const SAPLING_SHARD_HEIGHT: u8 = sapling::NOTE_COMMITMENT_TREE_DEPTH / 2; +/// An enumeration of constraints that can be applied when querying for nullifiers for notes +/// belonging to the wallet. pub enum NullifierQuery { Unspent, All, @@ -58,13 +60,6 @@ pub trait WalletRead { /// or a UUID. type NoteRef: Copy + Debug + Eq + Ord; - /// Backend-specific transaction identifier. - /// - /// For example, this might be a database identifier type - /// or a TxId if the backend is able to support that type - /// directly. - type TxRef: Copy + Debug + Eq + Ord; - /// Returns the minimum and maximum block heights for stored blocks. /// /// This will return `Ok(None)` if no block data is present in the database. @@ -189,14 +184,13 @@ pub trait WalletRead { /// Returns the memo for a note. /// - /// Implementations of this method must return an error if the note identifier - /// does not appear in the backing data store. Returns `Ok(None)` if the note - /// is known to the wallet but memo data has not yet been populated for that - /// note. - fn get_memo(&self, id_note: Self::NoteRef) -> Result, Self::Error>; + /// Returns `Ok(None)` if the note is known to the wallet but memo data has not yet been + /// populated for that note, or if the note identifier does not correspond to a note + /// that is known to the wallet. + fn get_memo(&self, note_id: NoteId) -> Result, Self::Error>; /// Returns a transaction. - fn get_transaction(&self, id_tx: Self::TxRef) -> Result; + fn get_transaction(&self, txid: TxId) -> Result; /// Returns the nullifiers for notes that the wallet is tracking, along with their associated /// account IDs, that are either unspent or have not yet been confirmed as spent (in that a @@ -206,7 +200,7 @@ pub trait WalletRead { query: NullifierQuery, ) -> Result, Self::Error>; - /// Return all unspent Sapling notes. + /// Return all unspent Sapling notes, excluding the specified note IDs. fn get_spendable_sapling_notes( &self, account: AccountId, @@ -380,13 +374,43 @@ pub struct SentTransaction<'a> { } /// A shielded transfer protocol supported by the wallet. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] pub enum ShieldedProtocol { /// The Sapling protocol Sapling, // TODO: Orchard } +/// A unique identifier for a shielded transaction output +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct NoteId { + txid: TxId, + protocol: ShieldedProtocol, + output_index: u32, +} + +impl NoteId { + pub fn new(txid: TxId, protocol: ShieldedProtocol, output_index: u32) -> Self { + Self { + txid, + protocol, + output_index, + } + } + + pub fn txid(&self) -> &TxId { + &self.txid + } + + pub fn protocol(&self) -> ShieldedProtocol { + self.protocol + } + + pub fn output_index(&self) -> u32 { + self.output_index + } +} + /// A value pool to which the wallet supports sending transaction outputs. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum PoolType { @@ -508,7 +532,7 @@ pub trait WalletWrite: WalletRead { fn put_blocks( &mut self, blocks: Vec>, - ) -> Result, Self::Error>; + ) -> Result<(), Self::Error>; /// Updates the wallet's view of the blockchain. /// @@ -520,14 +544,11 @@ pub trait WalletWrite: WalletRead { fn update_chain_tip(&mut self, tip_height: BlockHeight) -> Result<(), Self::Error>; /// Caches a decrypted transaction in the persistent wallet store. - fn store_decrypted_tx( - &mut self, - received_tx: DecryptedTransaction, - ) -> Result; + fn store_decrypted_tx(&mut self, received_tx: DecryptedTransaction) -> Result<(), Self::Error>; /// Saves information about a transaction that was constructed and sent by the wallet to the /// persistent wallet store. - fn store_sent_tx(&mut self, sent_tx: &SentTransaction) -> Result; + fn store_sent_tx(&mut self, sent_tx: &SentTransaction) -> Result<(), Self::Error>; /// Truncates the wallet database to the specified height. /// @@ -610,7 +631,7 @@ pub mod testing { use super::{ chain::CommitmentTreeRoot, scanning::ScanRange, BlockMetadata, DecryptedTransaction, - NullifierQuery, ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, + NoteId, NullifierQuery, ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }; @@ -635,7 +656,6 @@ pub mod testing { impl WalletRead for MockWalletDb { type Error = (); type NoteRef = u32; - type TxRef = TxId; fn block_height_extrema(&self) -> Result, Self::Error> { Ok(None) @@ -707,11 +727,11 @@ pub mod testing { Ok(Amount::zero()) } - fn get_memo(&self, _id_note: Self::NoteRef) -> Result, Self::Error> { + fn get_memo(&self, _id_note: NoteId) -> Result, Self::Error> { Ok(None) } - fn get_transaction(&self, _id_tx: Self::TxRef) -> Result { + fn get_transaction(&self, _txid: TxId) -> Result { Err(()) } @@ -790,8 +810,8 @@ pub mod testing { fn put_blocks( &mut self, _blocks: Vec>, - ) -> Result, Self::Error> { - Ok(vec![]) + ) -> Result<(), Self::Error> { + Ok(()) } fn update_chain_tip(&mut self, _tip_height: BlockHeight) -> Result<(), Self::Error> { @@ -801,15 +821,12 @@ pub mod testing { fn store_decrypted_tx( &mut self, _received_tx: DecryptedTransaction, - ) -> Result { - Ok(TxId::from_bytes([0u8; 32])) + ) -> Result<(), Self::Error> { + Ok(()) } - fn store_sent_tx( - &mut self, - _sent_tx: &SentTransaction, - ) -> Result { - Ok(TxId::from_bytes([0u8; 32])) + fn store_sent_tx(&mut self, _sent_tx: &SentTransaction) -> Result<(), Self::Error> { + Ok(()) } fn truncate_to_height(&mut self, _block_height: BlockHeight) -> Result<(), Self::Error> { diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 2d09231e8d..c202a597b0 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1,7 +1,7 @@ -use std::fmt::Debug; use std::{convert::Infallible, num::NonZeroU32}; use shardtree::{error::ShardTreeError, store::ShardStore, ShardTree}; +use zcash_primitives::transaction::TxId; use zcash_primitives::{ consensus::{self, BlockHeight, NetworkUpgrade}, memo::MemoBytes, @@ -203,7 +203,7 @@ pub fn create_spend_to_address( ovk_policy: OvkPolicy, min_confirmations: NonZeroU32, ) -> Result< - DbT::TxRef, + TxId, Error< ::Error, ::Error, @@ -306,7 +306,7 @@ pub fn spend( ovk_policy: OvkPolicy, min_confirmations: NonZeroU32, ) -> Result< - DbT::TxRef, + TxId, Error< ::Error, ::Error, @@ -317,7 +317,6 @@ pub fn spend( > where DbT: WalletWrite + WalletCommitmentTrees, - DbT::TxRef: Copy + Debug, DbT::NoteRef: Copy + Eq + Ord, ParamsT: consensus::Parameters + Clone, InputsT: InputSelector, @@ -441,7 +440,7 @@ pub fn create_proposed_transaction( min_confirmations: NonZeroU32, change_memo: Option, ) -> Result< - DbT::TxRef, + TxId, Error< ::Error, ::Error, @@ -452,7 +451,6 @@ pub fn create_proposed_transaction( > where DbT: WalletWrite + WalletCommitmentTrees, - DbT::TxRef: Copy + Debug, DbT::NoteRef: Copy + Eq + Ord, ParamsT: consensus::Parameters + Clone, FeeRuleT: FeeRule, @@ -670,7 +668,9 @@ where #[cfg(feature = "transparent-inputs")] utxos_spent: utxos.iter().map(|utxo| utxo.outpoint().clone()).collect(), }) - .map_err(Error::DataSource) + .map_err(Error::DataSource)?; + + Ok(tx.txid()) } /// Constructs a transaction that consumes available transparent UTXOs belonging to @@ -720,7 +720,7 @@ pub fn shield_transparent_funds( memo: &MemoBytes, min_confirmations: NonZeroU32, ) -> Result< - DbT::TxRef, + TxId, Error< ::Error, ::Error, diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 8700b10425..8b2da642a9 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -12,6 +12,7 @@ and this library adheres to Rust's notion of - A new default-enabled feature flag `multicore`. This allows users to disable multicore support by setting `default_features = false` on their `zcash_primitives`, `zcash_proofs`, and `zcash_client_sqlite` dependencies. +- `zcash_client_sqlite::ReceivedNoteId` - `zcash_client_sqlite::wallet::commitment_tree` A new module containing a sqlite-backed implementation of `shardtree::store::ShardStore`. @@ -30,6 +31,9 @@ and this library adheres to Rust's notion of ### Removed - The empty `wallet::transact` module has been removed. +- `zcash_client_sqlite::NoteId` has been replaced with `zcash_client_sqlite::ReceivedNoteId` + as the `SentNoteId` variant of is now unused following changes to + `zcash_client_backend::data_api::WalletRead`. ### Fixed - Fixed an off-by-one error in the `BlockSource` implementation for the SQLite-backed diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 5b92d9e3d5..60c96d831c 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -38,7 +38,7 @@ use maybe_rayon::{ }; use rusqlite::{self, Connection}; use secrecy::{ExposeSecret, SecretVec}; -use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, ops::Range, path::Path}; +use std::{borrow::Borrow, collections::HashMap, convert::AsRef, ops::Range, path::Path}; use incrementalmerkletree::Position; use shardtree::{error::ShardTreeError, ShardTree}; @@ -61,9 +61,9 @@ use zcash_client_backend::{ self, chain::{BlockSource, CommitmentTreeRoot}, scanning::{ScanPriority, ScanRange}, - BlockMetadata, DecryptedTransaction, NullifierQuery, PoolType, Recipient, ScannedBlock, - SentTransaction, ShieldedProtocol, WalletCommitmentTrees, WalletRead, WalletWrite, - SAPLING_SHARD_HEIGHT, + BlockMetadata, DecryptedTransaction, NoteId, NullifierQuery, PoolType, Recipient, + ScannedBlock, SentTransaction, ShieldedProtocol, WalletCommitmentTrees, WalletRead, + WalletWrite, SAPLING_SHARD_HEIGHT, }, keys::{UnifiedFullViewingKey, UnifiedSpendingKey}, proto::compact_formats::CompactBlock, @@ -97,22 +97,9 @@ pub(crate) const VERIFY_LOOKAHEAD: u32 = 10; pub(crate) const SAPLING_TABLES_PREFIX: &str = "sapling"; -/// A newtype wrapper for sqlite primary key values for the notes -/// table. +/// A newtype wrapper for received note identifiers. #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub enum NoteId { - SentNoteId(i64), - ReceivedNoteId(i64), -} - -impl fmt::Display for NoteId { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - NoteId::SentNoteId(id) => write!(f, "Sent Note {}", id), - NoteId::ReceivedNoteId(id) => write!(f, "Received Note {}", id), - } - } -} +pub struct ReceivedNoteId(pub(crate) i64); /// A newtype wrapper for sqlite primary key values for the utxos /// table. @@ -160,8 +147,7 @@ impl WalletDb { impl, P: consensus::Parameters> WalletRead for WalletDb { type Error = SqliteClientError; - type NoteRef = NoteId; - type TxRef = i64; + type NoteRef = ReceivedNoteId; fn block_height_extrema(&self) -> Result, Self::Error> { wallet::block_height_extrema(self.conn.borrow()).map_err(SqliteClientError::from) @@ -229,17 +215,17 @@ impl, P: consensus::Parameters> WalletRead for W wallet::get_balance_at(self.conn.borrow(), account, anchor_height) } - fn get_memo(&self, id_note: Self::NoteRef) -> Result, Self::Error> { - match id_note { - NoteId::SentNoteId(id_note) => wallet::get_sent_memo(self.conn.borrow(), id_note), - NoteId::ReceivedNoteId(id_note) => { - wallet::get_received_memo(self.conn.borrow(), id_note) - } + fn get_memo(&self, note_id: NoteId) -> Result, Self::Error> { + let sent_memo = wallet::get_sent_memo(self.conn.borrow(), note_id)?; + if sent_memo.is_some() { + Ok(sent_memo) + } else { + wallet::get_received_memo(self.conn.borrow(), note_id) } } - fn get_transaction(&self, id_tx: i64) -> Result { - wallet::get_transaction(self.conn.borrow(), &self.params, id_tx).map(|(_, tx)| tx) + fn get_transaction(&self, txid: TxId) -> Result { + wallet::get_transaction(self.conn.borrow(), &self.params, txid).map(|(_, tx)| tx) } fn get_sapling_nullifiers( @@ -404,7 +390,7 @@ impl WalletWrite for WalletDb fn put_blocks( &mut self, blocks: Vec>, - ) -> Result, Self::Error> { + ) -> Result<(), Self::Error> { self.transactionally(|wdb| { let start_positions = blocks.first().map(|block| { ( @@ -415,7 +401,6 @@ impl WalletWrite for WalletDb ), ) }); - let mut wallet_note_ids = vec![]; let mut sapling_commitments = vec![]; let mut last_scanned_height = None; let mut note_positions = vec![]; @@ -453,12 +438,7 @@ impl WalletWrite for WalletDb output.nf(), )?; - let received_note_id = wallet::sapling::put_received_note( - wdb.conn.0, output, tx_row, spent_in, - )?; - - // Save witness for note. - wallet_note_ids.push(received_note_id); + wallet::sapling::put_received_note(wdb.conn.0, output, tx_row, spent_in)?; } } @@ -535,7 +515,7 @@ impl WalletWrite for WalletDb )?; } - Ok(wallet_note_ids) + Ok(()) }) } @@ -546,10 +526,7 @@ impl WalletWrite for WalletDb Ok(()) } - fn store_decrypted_tx( - &mut self, - d_tx: DecryptedTransaction, - ) -> Result { + fn store_decrypted_tx(&mut self, d_tx: DecryptedTransaction) -> Result<(), Self::Error> { self.transactionally(|wdb| { let tx_ref = wallet::put_tx_data(wdb.conn.0, d_tx.tx, None, None)?; @@ -636,11 +613,11 @@ impl WalletWrite for WalletDb } } - Ok(tx_ref) + Ok(()) }) } - fn store_sent_tx(&mut self, sent_tx: &SentTransaction) -> Result { + fn store_sent_tx(&mut self, sent_tx: &SentTransaction) -> Result<(), Self::Error> { self.transactionally(|wdb| { let tx_ref = wallet::put_tx_data( wdb.conn.0, @@ -699,8 +676,7 @@ impl WalletWrite for WalletDb } } - // Return the row number of the transaction, so the caller can fetch it for sending. - Ok(tx_ref) + Ok(()) }) } diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 307eb59bf4..dc399a9ba3 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -70,7 +70,7 @@ use std::convert::TryFrom; use std::io::{self, Cursor}; use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; -use zcash_client_backend::data_api::ShieldedProtocol; +use zcash_client_backend::data_api::{NoteId, ShieldedProtocol}; use zcash_primitives::transaction::TransactionData; use zcash_primitives::{ @@ -474,20 +474,27 @@ pub(crate) fn get_balance_at( } } -/// Returns the memo for a received note. -/// -/// The note is identified by its row index in the `sapling_received_notes` table within the wdb -/// database. +/// Returns the memo for a received note, if the note is known to the wallet. pub(crate) fn get_received_memo( conn: &rusqlite::Connection, - id_note: i64, + note_id: NoteId, ) -> Result, SqliteClientError> { - let memo_bytes: Option> = conn.query_row( - "SELECT memo FROM sapling_received_notes - WHERE id_note = ?", - [id_note], - |row| row.get(0), - )?; + let memo_bytes: Option> = match note_id.protocol() { + ShieldedProtocol::Sapling => conn + .query_row( + "SELECT memo FROM sapling_received_notes + JOIN transactions ON sapling_received_notes.tx = transactions.id_tx + WHERE transactions.txid = :txid + AND sapling_received_notes.output_index = :output_index", + named_params![ + ":txid": note_id.txid().as_ref(), + ":output_index": note_id.output_index() + ], + |row| row.get(0), + ) + .optional()? + .flatten(), + }; memo_bytes .map(|b| { @@ -507,7 +514,7 @@ pub(crate) fn get_received_memo( pub(crate) fn get_transaction( conn: &rusqlite::Connection, params: &P, - id_tx: i64, + txid: TxId, ) -> Result<(BlockHeight, Transaction), SqliteClientError> { let (tx_bytes, block_height, expiry_height): ( Vec<_>, @@ -515,8 +522,8 @@ pub(crate) fn get_transaction( Option, ) = conn.query_row( "SELECT raw, block, expiry_height FROM transactions - WHERE id_tx = ?", - [id_tx], + WHERE txid = ?", + [txid.as_ref()], |row| { let h: Option = row.get(1)?; let expiry: Option = row.get(2)?; @@ -572,20 +579,27 @@ pub(crate) fn get_transaction( } } -/// Returns the memo for a sent note. -/// -/// The note is identified by its row index in the `sent_notes` table within the wdb -/// database. +/// Returns the memo for a sent note, if the sent note is known to the wallet. pub(crate) fn get_sent_memo( conn: &rusqlite::Connection, - id_note: i64, + note_id: NoteId, ) -> Result, SqliteClientError> { - let memo_bytes: Option> = conn.query_row( - "SELECT memo FROM sent_notes - WHERE id_note = ?", - [id_note], - |row| row.get(0), - )?; + let memo_bytes: Option> = conn + .query_row( + "SELECT memo FROM sent_notes + JOIN transactions ON sent_notes.tx = transactions.id_tx + WHERE transactions.txid = :txid + AND sent_notes.output_pool = :pool_code + AND sent_notes.output_index = :output_index", + named_params![ + ":txid": note_id.txid().as_ref(), + ":pool_code": pool_code(PoolType::Shielded(note_id.protocol())), + ":output_index": note_id.output_index() + ], + |row| row.get(0), + ) + .optional()? + .flatten(); memo_bytes .map(|b| { diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 540a8fcb10..a146425169 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -346,7 +346,7 @@ pub fn init_blocks_table( #[cfg(test)] #[allow(deprecated)] mod tests { - use rusqlite::{self, ToSql}; + use rusqlite::{self, named_params, ToSql}; use secrecy::Secret; use std::collections::HashMap; use tempfile::NamedTempFile; @@ -969,8 +969,11 @@ mod tests { let mut tx_bytes = vec![]; tx.write(&mut tx_bytes).unwrap(); wdb.conn.execute( - "INSERT INTO transactions (block, id_tx, txid, raw) VALUES (0, 0, '', ?)", - [&tx_bytes[..]], + "INSERT INTO transactions (block, id_tx, txid, raw) VALUES (0, 0, :txid, :tx_bytes)", + named_params![ + ":txid": tx.txid().as_ref(), + ":tx_bytes": &tx_bytes[..] + ], )?; wdb.conn.execute( "INSERT INTO sent_notes (tx, output_index, from_account, address, value) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs b/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs index 4cfc983827..582904359d 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs @@ -8,7 +8,7 @@ use rusqlite::named_params; use schemer_rusqlite::RusqliteMigration; use uuid::Uuid; use zcash_client_backend::{decrypt_transaction, keys::UnifiedFullViewingKey}; -use zcash_primitives::{consensus, zip32::AccountId}; +use zcash_primitives::{consensus, transaction::TxId, zip32::AccountId}; use crate::{ error::SqliteClientError, @@ -46,7 +46,9 @@ impl RusqliteMigration for Migration

{ fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { let mut stmt_raw_tx = transaction.prepare( - "SELECT DISTINCT sent_notes.tx, accounts.account, accounts.ufvk + "SELECT DISTINCT + transactions.id_tx, transactions.txid, + accounts.account, accounts.ufvk FROM sent_notes JOIN accounts ON sent_notes.from_account = accounts.account JOIN transactions ON transactions.id_tx = sent_notes.tx @@ -55,12 +57,13 @@ impl RusqliteMigration for Migration

{ let mut rows = stmt_raw_tx.query([])?; - let mut tx_sent_notes: BTreeMap> = + let mut tx_sent_notes: BTreeMap<(i64, TxId), HashMap> = BTreeMap::new(); while let Some(row) = rows.next()? { let id_tx: i64 = row.get(0)?; - let account: u32 = row.get(1)?; - let ufvk_str: String = row.get(2)?; + let txid = row.get(1).map(TxId::from_bytes)?; + let account: u32 = row.get(2)?; + let ufvk_str: String = row.get(3)?; let ufvk = UnifiedFullViewingKey::decode(&self.params, &ufvk_str).map_err(|e| { WalletMigrationError::CorruptedData(format!( "Could not decode unified full viewing key for account {}: {:?}", @@ -69,7 +72,7 @@ impl RusqliteMigration for Migration

{ })?; tx_sent_notes - .entry(id_tx) + .entry((id_tx, txid)) .or_default() .insert(AccountId::from(account), ufvk); } @@ -81,9 +84,9 @@ impl RusqliteMigration for Migration

{ AND output_index = :output_index", )?; - for (id_tx, ufvks) in tx_sent_notes { + for ((id_tx, txid), ufvks) in tx_sent_notes { let (block_height, tx) = - get_transaction(transaction, &self.params, id_tx).map_err(|err| match err { + get_transaction(transaction, &self.params, txid).map_err(|err| match err { SqliteClientError::CorruptedData(msg) => { WalletMigrationError::CorruptedData(msg) } diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 88cf3b2098..b987322010 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -18,7 +18,7 @@ use zcash_client_backend::{ DecryptedOutput, TransferType, }; -use crate::{error::SqliteClientError, NoteId}; +use crate::{error::SqliteClientError, ReceivedNoteId}; use super::memo_repr; @@ -81,8 +81,8 @@ impl ReceivedSaplingOutput for DecryptedOutput { } } -fn to_spendable_note(row: &Row) -> Result, SqliteClientError> { - let note_id = NoteId::ReceivedNoteId(row.get(0)?); +fn to_spendable_note(row: &Row) -> Result, SqliteClientError> { + let note_id = ReceivedNoteId(row.get(0)?); let diversifier = { let d: Vec<_> = row.get(1)?; if d.len() != 11 { @@ -130,8 +130,8 @@ pub(crate) fn get_spendable_sapling_notes( conn: &Connection, account: AccountId, anchor_height: BlockHeight, - exclude: &[NoteId], -) -> Result>, SqliteClientError> { + exclude: &[ReceivedNoteId], +) -> Result>, SqliteClientError> { let mut stmt_select_notes = conn.prepare_cached( "SELECT id_note, diversifier, value, rcm, commitment_tree_position FROM sapling_received_notes @@ -142,13 +142,7 @@ pub(crate) fn get_spendable_sapling_notes( AND id_note NOT IN rarray(:exclude)", )?; - let excluded: Vec = exclude - .iter() - .filter_map(|n| match n { - NoteId::ReceivedNoteId(i) => Some(Value::from(*i)), - NoteId::SentNoteId(_) => None, - }) - .collect(); + let excluded: Vec = exclude.iter().map(|n| Value::from(n.0)).collect(); let excluded_ptr = Rc::new(excluded); let notes = stmt_select_notes.query_and_then( @@ -168,8 +162,8 @@ pub(crate) fn select_spendable_sapling_notes( account: AccountId, target_value: Amount, anchor_height: BlockHeight, - exclude: &[NoteId], -) -> Result>, SqliteClientError> { + exclude: &[ReceivedNoteId], +) -> Result>, SqliteClientError> { // The goal of this SQL statement is to select the oldest notes until the required // value has been reached, and then fetch the witnesses at the desired height for the // selected notes. This is achieved in several steps: @@ -207,13 +201,7 @@ pub(crate) fn select_spendable_sapling_notes( FROM (SELECT * from eligible WHERE so_far >= :target_value LIMIT 1)", )?; - let excluded: Vec = exclude - .iter() - .filter_map(|n| match n { - NoteId::ReceivedNoteId(i) => Some(Value::from(*i)), - NoteId::SentNoteId(_) => None, - }) - .collect(); + let excluded: Vec = exclude.iter().map(|n| Value::from(n.0)).collect(); let excluded_ptr = Rc::new(excluded); let notes = stmt_select_notes.query_and_then( @@ -313,7 +301,7 @@ pub(crate) fn put_received_note( output: &T, tx_ref: i64, spent_in: Option, -) -> Result { +) -> Result<(), SqliteClientError> { let mut stmt_upsert_received_note = conn.prepare_cached( "INSERT INTO sapling_received_notes (tx, output_index, account, diversifier, value, rcm, memo, nf, is_change, spent, commitment_tree_position) @@ -339,8 +327,7 @@ pub(crate) fn put_received_note( memo = IFNULL(:memo, memo), is_change = IFNULL(:is_change, is_change), spent = IFNULL(:spent, spent), - commitment_tree_position = IFNULL(:commitment_tree_position, commitment_tree_position) - RETURNING id_note", + commitment_tree_position = IFNULL(:commitment_tree_position, commitment_tree_position)", )?; let rcm = output.note().rcm().to_repr(); @@ -362,10 +349,10 @@ pub(crate) fn put_received_note( ]; stmt_upsert_received_note - .query_row(sql_args, |row| { - row.get::<_, i64>(0).map(NoteId::ReceivedNoteId) - }) - .map_err(SqliteClientError::from) + .execute(sql_args) + .map_err(SqliteClientError::from)?; + + Ok(()) } #[cfg(test)] @@ -403,7 +390,7 @@ pub(crate) mod tests { create_proposed_transaction, create_spend_to_address, input_selection::GreedyInputSelector, propose_transfer, spend, }, - WalletRead, WalletWrite, + ShieldedProtocol, WalletRead, WalletWrite, }, decrypt_transaction, fees::{fixed, zip317, DustOutputPolicy}, @@ -566,13 +553,24 @@ pub(crate) mod tests { // Verify that the stored sent notes match what we're expecting let mut stmt_sent_notes = db_data .conn - .prepare("SELECT id_note FROM sent_notes WHERE tx = ?") + .prepare( + "SELECT output_index + FROM sent_notes + JOIN transactions ON transactions.id_tx = sent_notes.tx + WHERE transactions.txid = ?", + ) .unwrap(); let sent_note_ids = stmt_sent_notes - .query(rusqlite::params![sent_tx_id]) + .query(rusqlite::params![sent_tx_id.as_ref()]) .unwrap() - .mapped(|row| row.get::<_, i64>(0).map(NoteId::SentNoteId)) + .mapped(|row| { + Ok(NoteId::new( + sent_tx_id, + ShieldedProtocol::Sapling, + row.get(0)?, + )) + }) .collect::, _>>() .unwrap(); @@ -601,8 +599,11 @@ pub(crate) mod tests { assert!(found_sent_change_memo); assert!(found_sent_empty_memo); - // Check that querying for a nonexistent sent note returns an error - assert_matches!(db_data.get_memo(NoteId::SentNoteId(12345)), Err(_)); + // Check that querying for a nonexistent sent note returns None + assert_matches!( + db_data.get_memo(NoteId::new(sent_tx_id, ShieldedProtocol::Sapling, 12345)), + Ok(None) + ); } #[test] @@ -1101,7 +1102,7 @@ pub(crate) mod tests { let to = addr2.into(); let send_and_recover_with_policy = |db_data: &mut WalletDb, ovk_policy| { - let tx_row = create_spend_to_address( + let txid = create_spend_to_address( db_data, &tests::network(), test_prover(), @@ -1119,8 +1120,8 @@ pub(crate) mod tests { .conn .query_row( "SELECT raw FROM transactions - WHERE id_tx = ?", - [tx_row], + WHERE txid = ?", + [txid.as_ref()], |row| row.get(0), ) .unwrap(); From f602ec125d4de4b32e79ba583113ec57ff01c56d Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 7 Aug 2023 11:25:27 -0600 Subject: [PATCH 0096/1122] Apply suggestions from code review Co-authored-by: str4d --- zcash_client_backend/CHANGELOG.md | 3 ++- zcash_client_backend/src/data_api.rs | 11 ++++++++--- zcash_client_sqlite/src/wallet.rs | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 2b1602eb6d..c76b580eaf 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -37,7 +37,8 @@ and this library adheres to Rust's notion of - `WalletRead::TxRef` has been removed in favor of consistently using `TxId` instead. - `WalletRead::get_transaction` now takes a `TxId` as its argument. - `WalletWrite::{store_decrypted_tx, store_sent_tx}` now return `Result<(), Self::Error>` - as the `WalletRead::TxRef` associated type has been removed. Use `TxId` instead. + as the `WalletRead::TxRef` associated type has been removed. Use + `WalletRead::get_transaction` with the transaction's `TxId` instead. - `WalletRead::get_memo` now takes a `NoteId` as its argument instead of `Self::NoteRef` and returns `Result, Self::Error>` instead of `Result` in order to make representable wallet states where the full diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 2e0562257e..af11160315 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -386,11 +386,12 @@ pub enum ShieldedProtocol { pub struct NoteId { txid: TxId, protocol: ShieldedProtocol, - output_index: u32, + output_index: u16, } impl NoteId { - pub fn new(txid: TxId, protocol: ShieldedProtocol, output_index: u32) -> Self { + /// Constructs a new `NoteId` from its parts. + pub fn new(txid: TxId, protocol: ShieldedProtocol, output_index: u16) -> Self { Self { txid, protocol, @@ -398,15 +399,19 @@ impl NoteId { } } + /// Returns the ID of the transaction containing this note. pub fn txid(&self) -> &TxId { &self.txid } + /// Returns the shielded protocol used by this note. pub fn protocol(&self) -> ShieldedProtocol { self.protocol } - pub fn output_index(&self) -> u32 { + /// Returns the index of this note within its transaction's corresponding list of + /// shielded outputs. + pub fn output_index(&self) -> u16 { self.output_index } } diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index dc399a9ba3..ac60f2ced6 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -505,7 +505,7 @@ pub(crate) fn get_received_memo( .transpose() } -/// Looks up a transaction by its internal database identifier. +/// Looks up a transaction by its [`TxId`]. /// /// Returns the decoded transaction, along with the block height that was used in its decoding. /// This is either the block height at which the transaction was mined, or the expiry height if the From 1c4e63718f93ba98d6de77cb3a9c5915ad20afd5 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 7 Aug 2023 12:24:05 -0600 Subject: [PATCH 0097/1122] Add `Display` impl for `ReceivedNoteId` --- zcash_client_sqlite/src/lib.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 60c96d831c..4a9cc7b44e 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -38,7 +38,7 @@ use maybe_rayon::{ }; use rusqlite::{self, Connection}; use secrecy::{ExposeSecret, SecretVec}; -use std::{borrow::Borrow, collections::HashMap, convert::AsRef, ops::Range, path::Path}; +use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, ops::Range, path::Path}; use incrementalmerkletree::Position; use shardtree::{error::ShardTreeError, ShardTree}; @@ -101,6 +101,14 @@ pub(crate) const SAPLING_TABLES_PREFIX: &str = "sapling"; #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct ReceivedNoteId(pub(crate) i64); +impl fmt::Display for ReceivedNoteId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + ReceivedNoteId(id) => write!(f, "Received Note {}", id), + } + } +} + /// A newtype wrapper for sqlite primary key values for the utxos /// table. #[derive(Debug, Copy, Clone, PartialEq, Eq)] From 44abd3450bb19e50c852ff38ef9585b2a2b96e8b Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 9 Aug 2023 17:31:23 +0000 Subject: [PATCH 0098/1122] Require `BlockSource::with_blocks` fail on non-existent `from_height` Previously this was not clearly specified, and the implementations in `zcash_client_sqlite` behaved similarly to when `from_height = None`. Closes zcash/librustzcash#892. --- zcash_client_backend/CHANGELOG.md | 3 +- zcash_client_backend/src/data_api/chain.rs | 2 ++ zcash_client_sqlite/CHANGELOG.md | 8 ++++- zcash_client_sqlite/src/chain.rs | 34 ++++++++++++++++++++++ zcash_client_sqlite/src/error.rs | 3 ++ zcash_client_sqlite/src/lib.rs | 8 +++++ 6 files changed, 56 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index c76b580eaf..bcd1b6b5dc 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -50,7 +50,8 @@ and this library adheres to Rust's notion of permits the caller to control the starting position of the scan range. In addition, the `limit` parameter is now required and has type `usize`. - `chain::BlockSource::with_blocks` now takes its limit as an `Option` - instead of `Option`. + instead of `Option`. It is also now required to return an error if + `from_height` is set to a block that does not exist in `self`. - A new `CommitmentTree` variant has been added to `data_api::error::Error` - `wallet::{create_spend_to_address, create_proposed_transaction, shield_transparent_funds}` all now require that `WalletCommitmentTrees` be diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index e6c5125178..60d0cff1d9 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -199,6 +199,8 @@ pub trait BlockSource { /// `from_height`, applying the provided callback to each block. If `from_height` /// is `None` then scanning will begin at the first available block. /// + /// Returns an error if `from_height` is set to a block that does not exist in `self`. + /// /// * `WalletErrT`: the types of errors produced by the wallet operations performed /// as part of processing each row. /// * `NoteRefT`: the type of note identifiers in the wallet data store, for use in diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 8b2da642a9..02a9f57424 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -25,7 +25,11 @@ and this library adheres to Rust's notion of a note could be spent with fewer than `min_confirmations` confirmations if the wallet did not contain enough observed blocks to satisfy the `min_confirmations` value specified; this situation is now treated as an error. -- A `BlockConflict` variant has been added to `zcash_client_sqlite::error::SqliteClientError` +- `zcash_client_sqlite::error::SqliteClientError` has new error variants: + - `SqliteClientError::BlockConflict` + - `SqliteClientError::CacheMiss` +- `zcash_client_backend::FsBlockDbError` has a new error variant: + - `FsBlockDbError::CacheMiss` - `zcash_client_sqlite::FsBlockDb::write_block_metadata` now overwrites any existing metadata entries that have the same height as a new entry. @@ -39,6 +43,8 @@ and this library adheres to Rust's notion of - Fixed an off-by-one error in the `BlockSource` implementation for the SQLite-backed `BlockDb` block database which could result in blocks being skipped at the start of scan ranges. +- `zcash_client_sqlite::{BlockDb, FsBlockDb}::with_blocks` now return an error + if `from_height` is set to a block height that does not exist in the cache. - `WalletDb::get_transaction` no longer returns an error when called on a transaction that has not yet been mined, unless the transaction's consensus branch ID cannot be determined by other means. diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 7a2396dfe1..c66cde05d3 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -58,8 +58,20 @@ where ]) .map_err(to_chain_error)?; + // Only look for the `from_height` in the scanned blocks if it is set. + let mut from_height_found = from_height.is_none(); while let Some(row) = rows.next().map_err(to_chain_error)? { let height = BlockHeight::from_u32(row.get(0).map_err(to_chain_error)?); + if !from_height_found { + // We will only perform this check on the first row. + let from_height = from_height.expect("can only reach here if set"); + if from_height != height { + return Err(to_chain_error(SqliteClientError::CacheMiss(from_height))); + } else { + from_height_found = true; + } + } + let data: Vec = row.get(1).map_err(to_chain_error)?; let block = CompactBlock::decode(&data[..]).map_err(to_chain_error)?; if block.height() != height { @@ -73,6 +85,11 @@ where with_row(block)?; } + if !from_height_found { + let from_height = from_height.expect("can only reach here if set"); + return Err(to_chain_error(SqliteClientError::CacheMiss(from_height))); + } + Ok(()) } @@ -260,8 +277,20 @@ where ) .map_err(to_chain_error)?; + // Only look for the `from_height` in the scanned blocks if it is set. + let mut from_height_found = from_height.is_none(); for row_result in rows { let cbr = row_result.map_err(to_chain_error)?; + if !from_height_found { + // We will only perform this check on the first row. + let from_height = from_height.expect("can only reach here if set"); + if from_height != cbr.height { + return Err(to_chain_error(FsBlockDbError::CacheMiss(from_height))); + } else { + from_height_found = true; + } + } + let mut block_file = File::open(cbr.block_file_path(&cache.blocks_dir)).map_err(to_chain_error)?; let mut block_data = vec![]; @@ -282,6 +311,11 @@ where with_block(block)?; } + if !from_height_found { + let from_height = from_height.expect("can only reach here if set"); + return Err(to_chain_error(FsBlockDbError::CacheMiss(from_height))); + } + Ok(()) } diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index dec1aa7091..618b1d1c01 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -85,6 +85,8 @@ pub enum SqliteClientError { /// An error occurred in inserting data into or accessing data from one of the wallet's note /// commitment trees. CommitmentTree(ShardTreeError), + + CacheMiss(BlockHeight), } impl error::Error for SqliteClientError { @@ -128,6 +130,7 @@ impl fmt::Display for SqliteClientError { #[cfg(feature = "transparent-inputs")] SqliteClientError::AddressNotRecognized(_) => write!(f, "The address associated with a received txo is not identifiable as belonging to the wallet."), SqliteClientError::CommitmentTree(err) => write!(f, "An error occurred accessing or updating note commitment tree data: {}.", err), + SqliteClientError::CacheMiss(height) => write!(f, "Requested height {} does not exist in the block cache.", height), } } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 4a9cc7b44e..5cc7cd49b7 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -883,6 +883,7 @@ pub enum FsBlockDbError { InvalidBlockstoreRoot(PathBuf), InvalidBlockPath(PathBuf), CorruptedData(String), + CacheMiss(BlockHeight), } #[cfg(feature = "unstable")] @@ -1047,6 +1048,13 @@ impl std::fmt::Display for FsBlockDbError { e, ) } + FsBlockDbError::CacheMiss(height) => { + write!( + f, + "Requested height {} does not exist in the block cache", + height + ) + } } } } From d3fc9670d57d441fadcf2885026bf2cc1fdc6ca7 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 9 Aug 2023 17:35:42 +0000 Subject: [PATCH 0099/1122] zcash_client_backend: Fix `scan_cached_blocks` panic with genesis block If `from_height = 0` and `limit > 1`, then the conditional being used to initialise `continuity_check_metadata` was previously running for every block, and would therefore panic while processing the block at height 1. --- zcash_client_backend/src/data_api/chain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 60d0cff1d9..08ee524f72 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -302,7 +302,7 @@ where return Err(Error::Scan(scan_error)); } - if from_height == BlockHeight::from(0) { + if block.height() == BlockHeight::from(0) { // We can always derive a valid `continuity_check_metadata` for the // genesis block, even if the block source doesn't have // `sapling_commitment_tree_size`. So briefly set it to a dummy value that From dee43854cb69fe428d4779eda7202d7744de0fb5 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 11 Aug 2023 16:41:38 -0600 Subject: [PATCH 0100/1122] zcash_client_sqlite: Ensure that target and anchor heights are relative to the chain tip. Prior to the scan-before-sync changes, the wallet was able to assume that the maximum scanned block height at the time of the spend was within a few blocks of the chain tip. However, under linear scanning after the spend-before-sync changes this invariant no longer holds, resulting in a situation where in linear sync conditions the wallet could attempt to create transactions with already-past expiry heights. This change separates the notion of "chain tip" from "max scanned height", relying upon the `scan_queue` table to maintain the wallet's view of the consensus chain height and using information from the `blocks` table only in situations where the latest and/or earliest scanned height is required. As part of this change, the `WalletRead` interface is also modified to disambiguate these concepts. --- zcash_client_backend/CHANGELOG.md | 16 ++-- zcash_client_backend/src/data_api.rs | 58 +++++------- zcash_client_backend/src/data_api/wallet.rs | 4 +- zcash_client_sqlite/src/chain.rs | 2 +- zcash_client_sqlite/src/lib.rs | 23 ++++- zcash_client_sqlite/src/wallet.rs | 62 ++++++++++++- zcash_client_sqlite/src/wallet/sapling.rs | 97 ++++++++------------- zcash_client_sqlite/src/wallet/scanning.rs | 8 +- 8 files changed, 153 insertions(+), 117 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index bcd1b6b5dc..6980d35ef9 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -17,7 +17,7 @@ and this library adheres to Rust's notion of - `ScannedBlock` - `ShieldedProtocol` - `WalletCommitmentTrees` - - `WalletRead::{block_metadata, block_fully_scanned, suggest_scan_ranges}` + - `WalletRead::{chain_height, block_metadata, block_fully_scanned, suggest_scan_ranges}` - `WalletWrite::{put_blocks, update_chain_tip}` - `chain::CommitmentTreeRoot` - `scanning` A new module containing types required for `suggest_scan_ranges` @@ -89,10 +89,13 @@ and this library adheres to Rust's notion of ### Removed - `zcash_client_backend::data_api`: - - `WalletRead::get_all_nullifiers` - - `WalletRead::{get_commitment_tree, get_witnesses}` have been removed - without replacement. The utility of these methods is now subsumed - by those available from the `WalletCommitmentTrees` trait. + - `WalletRead::block_height_extrema` has been removed. Use `chain_height` + instead to obtain the wallet's view of the chain tip instead, or + `suggest_scan_ranges` to obtain information about blocks that need to be + scanned. + - `WalletRead::{get_all_nullifiers, get_commitment_tree, get_witnesses}` have + been removed without replacement. The utility of these methods is now + subsumed by those available from the `WalletCommitmentTrees` trait. - `WalletWrite::advance_by_block` (use `WalletWrite::put_blocks` instead). - `PrunedBlock` has been replaced by `ScannedBlock` - `testing::MockWalletDb`, which is available under the `test-dependencies` @@ -107,6 +110,9 @@ and this library adheres to Rust's notion of have been removed as individual incremental witnesses are no longer tracked on a per-note basis. The global note commitment tree for the wallet should be used to obtain witnesses for spend operations instead. +- Default implementations of `zcash_client_backend::data_api::WalletRead::{ + get_target_and_anchor_heights, get_max_height_hash + }` have been removed. These should be implemented in a backend-specific fashion. ## [0.9.0] - 2023-04-28 diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index af11160315..d1711375f7 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -1,6 +1,5 @@ //! Interfaces for wallet data persistence & low-level wallet utilities. -use std::cmp; use std::collections::HashMap; use std::fmt::Debug; use std::num::NonZeroU32; @@ -60,10 +59,11 @@ pub trait WalletRead { /// or a UUID. type NoteRef: Copy + Debug + Eq + Ord; - /// Returns the minimum and maximum block heights for stored blocks. + /// Returns the height of the chain as known to the wallet as of the most recent call to + /// [`WalletWrite::update_chain_tip`]. /// - /// This will return `Ok(None)` if no block data is present in the database. - fn block_height_extrema(&self) -> Result, Self::Error>; + /// This will return `Ok(None)` if the height of the current consensus chain tip is unknown. + fn chain_height(&self) -> Result, Self::Error>; /// Returns the available block metadata for the block at the specified height, if any. fn block_metadata(&self, height: BlockHeight) -> Result, Self::Error>; @@ -98,22 +98,7 @@ pub trait WalletRead { fn get_target_and_anchor_heights( &self, min_confirmations: NonZeroU32, - ) -> Result, Self::Error> { - self.block_height_extrema().map(|heights| { - heights.map(|(min_height, max_height)| { - let target_height = max_height + 1; - - // Select an anchor min_confirmations back from the target block, - // unless that would be before the earliest block we have. - let anchor_height = BlockHeight::from(cmp::max( - u32::from(target_height).saturating_sub(min_confirmations.into()), - u32::from(min_height), - )); - - (target_height, anchor_height) - }) - }) - } + ) -> Result, Self::Error>; /// Returns the minimum block height corresponding to an unspent note in the wallet. fn get_min_unspent_height(&self) -> Result, Self::Error>; @@ -123,22 +108,10 @@ pub trait WalletRead { /// is not found in the database. fn get_block_hash(&self, block_height: BlockHeight) -> Result, Self::Error>; - /// Returns the block hash for the block at the maximum height known - /// in stored data. + /// Returns the block height and hash for the block at the maximum scanned block height. /// - /// This will return `Ok(None)` if no block data is present in the database. - fn get_max_height_hash(&self) -> Result, Self::Error> { - self.block_height_extrema() - .and_then(|extrema_opt| { - extrema_opt - .map(|(_, max_height)| { - self.get_block_hash(max_height) - .map(|hash_opt| hash_opt.map(move |hash| (max_height, hash))) - }) - .transpose() - }) - .map(|oo| oo.flatten()) - } + /// This will return `Ok(None)` if no blocks have been scanned. + fn get_max_height_hash(&self) -> Result, Self::Error>; /// Returns the block height in which the specified transaction was mined, or `Ok(None)` if the /// transaction is not in the main chain. @@ -613,7 +586,7 @@ pub mod testing { use incrementalmerkletree::Address; use secrecy::{ExposeSecret, SecretVec}; use shardtree::{error::ShardTreeError, store::memory::MemoryShardStore, ShardTree}; - use std::{collections::HashMap, convert::Infallible}; + use std::{collections::HashMap, convert::Infallible, num::NonZeroU32}; use zcash_primitives::{ block::BlockHash, @@ -662,7 +635,14 @@ pub mod testing { type Error = (); type NoteRef = u32; - fn block_height_extrema(&self) -> Result, Self::Error> { + fn chain_height(&self) -> Result, Self::Error> { + Ok(None) + } + + fn get_target_and_anchor_heights( + &self, + _min_confirmations: NonZeroU32, + ) -> Result, Self::Error> { Ok(None) } @@ -692,6 +672,10 @@ pub mod testing { Ok(None) } + fn get_max_height_hash(&self) -> Result, Self::Error> { + Ok(None) + } + fn get_tx_height(&self, _txid: TxId) -> Result, Self::Error> { Ok(None) } diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index c202a597b0..da820e7298 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -66,9 +66,7 @@ where // for mempool transactions. let height = data .get_tx_height(tx.txid())? - .or(data - .block_height_extrema()? - .map(|(_, max_height)| max_height + 1)) + .or(data.chain_height()?.map(|max_height| max_height + 1)) .or_else(|| params.activation_height(NetworkUpgrade::Sapling)) .expect("Sapling activation height must be known."); diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index c66cde05d3..f95f6b63ad 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -370,7 +370,7 @@ mod tests { let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); // Empty chain should return None - assert_matches!(db_data.get_max_height_hash(), Ok(None)); + assert_matches!(db_data.chain_height(), Ok(None)); // Create a fake CompactBlock sending value to the address let (cb, _) = fake_compact_block( diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 5cc7cd49b7..4935e2c5c9 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -38,7 +38,10 @@ use maybe_rayon::{ }; use rusqlite::{self, Connection}; use secrecy::{ExposeSecret, SecretVec}; -use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, ops::Range, path::Path}; +use std::{ + borrow::Borrow, collections::HashMap, convert::AsRef, fmt, num::NonZeroU32, ops::Range, + path::Path, +}; use incrementalmerkletree::Position; use shardtree::{error::ShardTreeError, ShardTree}; @@ -157,8 +160,10 @@ impl, P: consensus::Parameters> WalletRead for W type Error = SqliteClientError; type NoteRef = ReceivedNoteId; - fn block_height_extrema(&self) -> Result, Self::Error> { - wallet::block_height_extrema(self.conn.borrow()).map_err(SqliteClientError::from) + fn chain_height(&self) -> Result, Self::Error> { + wallet::scan_queue_extrema(self.conn.borrow()) + .map(|h| h.map(|(_, max)| max)) + .map_err(SqliteClientError::from) } fn block_metadata(&self, height: BlockHeight) -> Result, Self::Error> { @@ -174,6 +179,14 @@ impl, P: consensus::Parameters> WalletRead for W .map_err(SqliteClientError::from) } + fn get_target_and_anchor_heights( + &self, + min_confirmations: NonZeroU32, + ) -> Result, Self::Error> { + wallet::get_target_and_anchor_heights(self.conn.borrow(), min_confirmations) + .map_err(SqliteClientError::from) + } + fn get_min_unspent_height(&self) -> Result, Self::Error> { wallet::get_min_unspent_height(self.conn.borrow()).map_err(SqliteClientError::from) } @@ -182,6 +195,10 @@ impl, P: consensus::Parameters> WalletRead for W wallet::get_block_hash(self.conn.borrow(), block_height).map_err(SqliteClientError::from) } + fn get_max_height_hash(&self) -> Result, Self::Error> { + wallet::get_max_height_hash(self.conn.borrow()).map_err(SqliteClientError::from) + } + fn get_tx_height(&self, txid: TxId) -> Result, Self::Error> { wallet::get_tx_height(self.conn.borrow(), txid).map_err(SqliteClientError::from) } diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index ac60f2ced6..9de950997f 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -65,9 +65,11 @@ //! - `memo` the shielded memo associated with the output, if any. use rusqlite::{self, named_params, OptionalExtension, ToSql}; +use std::cmp; use std::collections::HashMap; use std::convert::TryFrom; use std::io::{self, Cursor}; +use std::num::NonZeroU32; use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; use zcash_client_backend::data_api::{NoteId, ShieldedProtocol}; @@ -623,6 +625,45 @@ pub(crate) fn block_height_extrema( }) } +/// Returns the minimum and maximum heights of blocks in the chain which may be scanned. +pub(crate) fn scan_queue_extrema( + conn: &rusqlite::Connection, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT MIN(block_range_start), MAX(block_range_end) FROM scan_queue", + [], + |row| { + let min_height: Option = row.get(0)?; + let max_height: Option = row.get(1)?; + + // Scan ranges are end-exclusive, so we subtract 1 from `max_height` to obtain the + // height of the last known chain tip; + Ok(min_height + .map(BlockHeight::from) + .zip(max_height.map(|h| BlockHeight::from(h.saturating_sub(1))))) + }, + ) +} + +pub(crate) fn get_target_and_anchor_heights( + conn: &rusqlite::Connection, + min_confirmations: NonZeroU32, +) -> Result, rusqlite::Error> { + scan_queue_extrema(conn).map(|heights| { + heights.map(|(min_height, max_height)| { + let target_height = max_height + 1; + // Select an anchor min_confirmations back from the target block, + // unless that would be before the earliest block we have. + let anchor_height = BlockHeight::from(cmp::max( + u32::from(target_height).saturating_sub(min_confirmations.into()), + u32::from(min_height), + )); + + (target_height, anchor_height) + }) + }) +} + fn parse_block_metadata( row: (BlockHeight, Vec, Option, Vec), ) -> Result { @@ -765,6 +806,21 @@ pub(crate) fn get_block_hash( .optional() } +pub(crate) fn get_max_height_hash( + conn: &rusqlite::Connection, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT height, hash FROM blocks ORDER BY height DESC LIMIT 1", + [], + |row| { + let height = row.get::<_, u32>(0).map(BlockHeight::from)?; + let row_data = row.get::<_, Vec<_>>(1)?; + Ok((height, BlockHash::from_slice(&row_data))) + }, + ) + .optional() +} + /// Gets the height to which the database must be truncated if any truncation that would remove a /// number of blocks greater than the pruning height is attempted. pub(crate) fn get_min_unspent_height( @@ -772,9 +828,9 @@ pub(crate) fn get_min_unspent_height( ) -> Result, SqliteClientError> { conn.query_row( "SELECT MIN(tx.block) - FROM sapling_received_notes n - JOIN transactions tx ON tx.id_tx = n.tx - WHERE n.spent IS NULL", + FROM sapling_received_notes n + JOIN transactions tx ON tx.id_tx = n.tx + WHERE n.spent IS NULL", [], |row| { row.get(0) diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index b987322010..2d0499b3e9 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -368,12 +368,14 @@ pub(crate) mod tests { use zcash_primitives::{ block::BlockHash, - consensus::{BlockHeight, BranchId}, + consensus::BranchId, legacy::TransparentAddress, memo::Memo, - sapling::{note_encryption::try_sapling_output_recovery, prover::TxProver}, + sapling::{ + note_encryption::try_sapling_output_recovery, prover::TxProver, Note, PaymentAddress, + }, transaction::{ - components::Amount, + components::{amount::BalanceError, Amount}, fees::{fixed::FeeRule as FixedFeeRule, zip317::FeeRule as Zip317FeeRule}, Transaction, }, @@ -388,7 +390,8 @@ pub(crate) mod tests { error::Error, wallet::{ create_proposed_transaction, create_spend_to_address, - input_selection::GreedyInputSelector, propose_transfer, spend, + input_selection::{GreedyInputSelector, GreedyInputSelectorError}, + propose_transfer, spend, }, ShieldedProtocol, WalletRead, WalletWrite, }, @@ -401,15 +404,13 @@ pub(crate) mod tests { use crate::{ chain::init::init_cache_database, + error::SqliteClientError, tests::{ self, fake_compact_block, insert_into_cache, network, sapling_activation_height, AddressType, }, - wallet::{ - get_balance, get_balance_at, - init::{init_blocks_table, init_wallet_db}, - }, - AccountId, BlockDb, NoteId, WalletDb, + wallet::{commitment_tree, get_balance, get_balance_at, init::init_wallet_db}, + AccountId, BlockDb, NoteId, ReceivedNoteId, WalletDb, }; #[cfg(feature = "transparent-inputs")] @@ -651,50 +652,13 @@ pub(crate) mod tests { let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); let to = dfvk.default_address().1.into(); - // We cannot do anything if we aren't synchronised - assert_matches!( - create_spend_to_address( - &mut db_data, - &tests::network(), - test_prover(), - &usk, - &to, - Amount::from_u64(1).unwrap(), - None, - OvkPolicy::Sender, - NonZeroU32::new(1).unwrap(), - ), - Err(data_api::error::Error::ScanRequired) - ); - } - - #[test] - fn create_to_address_fails_on_insufficient_balance() { - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); - init_wallet_db(&mut db_data, None).unwrap(); - init_blocks_table( - &mut db_data, - BlockHeight::from(1u32), - BlockHash([1; 32]), - 1, - &[0x0, 0x0, 0x0], - ) - .unwrap(); - - // Add an account to the wallet - let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = db_data.create_account(&seed).unwrap(); - let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); - let to = dfvk.default_address().1.into(); - // Account balance should be zero assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), Amount::zero() ); - // We cannot spend anything + // We cannot do anything if we aren't synchronised assert_matches!( create_spend_to_address( &mut db_data, @@ -707,11 +671,7 @@ pub(crate) mod tests { OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), ), - Err(data_api::error::Error::InsufficientFunds { - available, - required - }) - if available == Amount::zero() && required == Amount::from_u64(10001).unwrap() + Err(data_api::error::Error::ScanRequired) ); } @@ -1101,7 +1061,19 @@ pub(crate) mod tests { let addr2 = extsk2.default_address().1; let to = addr2.into(); - let send_and_recover_with_policy = |db_data: &mut WalletDb, ovk_policy| { + #[allow(clippy::type_complexity)] + let send_and_recover_with_policy = |db_data: &mut WalletDb, + ovk_policy| + -> Result< + Option<(Note, PaymentAddress, MemoBytes)>, + Error< + SqliteClientError, + commitment_tree::Error, + GreedyInputSelectorError, + Infallible, + ReceivedNoteId, + >, + > { let txid = create_spend_to_address( db_data, &tests::network(), @@ -1112,8 +1084,7 @@ pub(crate) mod tests { None, ovk_policy, NonZeroU32::new(1).unwrap(), - ) - .unwrap(); + )?; // Fetch the transaction from the database let raw_tx: Vec<_> = db_data @@ -1137,18 +1108,19 @@ pub(crate) mod tests { ); if result.is_some() { - return result; + return Ok(result); } } - None + Ok(None) }; // Send some of the funds to another address, keeping history. // The recipient output is decryptable by the sender. - let (_, recovered_to, _) = - send_and_recover_with_policy(&mut db_data, OvkPolicy::Sender).unwrap(); - assert_eq!(&recovered_to, &addr2); + assert_matches!( + send_and_recover_with_policy(&mut db_data, OvkPolicy::Sender), + Ok(Some((_, recovered_to, _))) if recovered_to == addr2 + ); // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 42 (that don't send us funds) // so that the first transaction expires @@ -1175,7 +1147,10 @@ pub(crate) mod tests { // Send the funds again, discarding history. // Neither transaction output is decryptable by the sender. - assert!(send_and_recover_with_policy(&mut db_data, OvkPolicy::Discard).is_none()); + assert_matches!( + send_and_recover_with_policy(&mut db_data, OvkPolicy::Discard), + Ok(None) + ); } #[test] diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index d3f456e1ab..83781837ac 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -619,12 +619,12 @@ pub(crate) fn update_chain_tip( params: &P, new_tip: BlockHeight, ) -> Result<(), SqliteClientError> { - // Read the previous tip height from the blocks table. + // Read the previous max scanned height from the blocks table let prior_tip = block_height_extrema(conn)?.map(|(_, prior_tip)| prior_tip); - // If the chain tip is below the prior tip height, then the caller has caught the - // chain in the middle of a reorg. Do nothing; the caller will continue using the old - // scan ranges and either: + // If the chain tip is below the prior max scanned height, then the caller has caught + // the chain in the middle of a reorg. Do nothing; the caller will continue using the + // old scan ranges and either: // - encounter an error trying to fetch the blocks (and thus trigger the same handling // logic as if this happened with the old linear scanning code); or // - encounter a discontinuity error in `scan_cached_blocks`, at which point they will From 50ea2a5b0f33a049e7c12f39b6b60c4e11232dff Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 9 Aug 2023 13:54:07 -0600 Subject: [PATCH 0101/1122] zcash_client_sqlite: Only select notes for which witnesses can be constructed. This change modifies the implementation of `get_spendable_sapling_notes` and `select_spendable_sapling_notes` to only return notes at positions where the associated note commitment tree shard has been fully scanned. This is slightly more conservative than it needs to be, because there could be cases where witnesses imported into the tree in the `shardtree_support` migration cover the complete range of a subtree (and hence that subtree doesn't need to be re-scanned). However, we can't detect or depend upon that condition in general without attempting to create a witness for each note retrieved. A possible alternative to this approach would be to not bound our query results on the requested total, and instead attempt to construct a witness for each note we retrieve, skipping the note if we cannot construct a witness. However, given that accessing the note commitment tree can be a costly operation requiring nontrivial deserialization costs, the more conservative database-oriented approach is perhaps better. --- zcash_client_sqlite/src/wallet/init.rs | 40 ++++++++-- .../src/wallet/init/migrations.rs | 6 ++ .../v_sapling_shard_unscanned_ranges.rs | 75 +++++++++++++++++++ zcash_client_sqlite/src/wallet/sapling.rs | 36 ++++++--- 4 files changed, 139 insertions(+), 18 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index a146425169..da9657ca45 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -353,14 +353,14 @@ mod tests { use zcash_client_backend::{ address::RecipientAddress, - data_api::WalletRead, + data_api::{scanning::ScanPriority, WalletRead}, encoding::{encode_extended_full_viewing_key, encode_payment_address}, keys::{sapling, UnifiedFullViewingKey, UnifiedSpendingKey}, }; use zcash_primitives::{ block::BlockHash, - consensus::{BlockHeight, BranchId, Parameters}, + consensus::{BlockHeight, BranchId, NetworkUpgrade, Parameters}, transaction::{TransactionData, TxVersion}, zip32::sapling::ExtendedFullViewingKey, }; @@ -368,6 +368,7 @@ mod tests { use crate::{ error::SqliteClientError, tests::{self, network}, + wallet::scanning::priority_code, AccountId, WalletDb, }; @@ -558,6 +559,33 @@ mod tests { } let expected_views = vec![ + // v_sapling_shard_unscanned_ranges + format!( + "CREATE VIEW v_sapling_shard_unscanned_ranges AS + SELECT + shard.shard_index, + shard.shard_index << 16 AS start_position, + (shard.shard_index + 1) << 16 AS end_position_exclusive, + IFNULL(prev_shard.subtree_end_height, {}) AS subtree_start_height, + shard.subtree_end_height AS subtree_end_height, + shard.contains_marked, + scan_queue.block_range_start, + scan_queue.block_range_end, + scan_queue.priority + FROM sapling_tree_shards shard + LEFT OUTER JOIN sapling_tree_shards prev_shard + ON shard.shard_index = prev_shard.shard_index + 1 + INNER JOIN scan_queue ON + (scan_queue.block_range_start BETWEEN subtree_start_height AND shard.subtree_end_height) OR + ((scan_queue.block_range_end - 1) BETWEEN subtree_start_height AND shard.subtree_end_height) OR + ( + scan_queue.block_range_start <= prev_shard.subtree_end_height + AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height + ) + WHERE scan_queue.priority != {}", + u32::from(tests::network().activation_height(NetworkUpgrade::Sapling).unwrap()), + priority_code(&ScanPriority::Scanned), + ), // v_transactions "CREATE VIEW v_transactions AS WITH @@ -649,7 +677,7 @@ mod tests { LEFT JOIN sent_note_counts ON sent_note_counts.account_id = notes.account_id AND sent_note_counts.id_tx = notes.id_tx - GROUP BY notes.account_id, transactions.id_tx", + GROUP BY notes.account_id, transactions.id_tx".to_owned(), // v_tx_outputs "CREATE VIEW v_tx_outputs AS SELECT sapling_received_notes.tx AS id_tx, @@ -693,7 +721,7 @@ mod tests { ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = (sapling_received_notes.tx, 2, sapling_received_notes.output_index) WHERE sapling_received_notes.is_change IS NULL - OR sapling_received_notes.is_change = 0" + OR sapling_received_notes.is_change = 0".to_owned(), ]; let mut views_query = db_data @@ -706,7 +734,7 @@ mod tests { let sql: String = row.get(0).unwrap(); assert_eq!( re.replace_all(&sql, " "), - re.replace_all(expected_views[expected_idx], " ") + re.replace_all(&expected_views[expected_idx], " ") ); expected_idx += 1; } @@ -971,7 +999,7 @@ mod tests { wdb.conn.execute( "INSERT INTO transactions (block, id_tx, txid, raw) VALUES (0, 0, :txid, :tx_bytes)", named_params![ - ":txid": tx.txid().as_ref(), + ":txid": tx.txid().as_ref(), ":tx_bytes": &tx_bytes[..] ], )?; diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 25d29c68ca..834960ec49 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -9,6 +9,7 @@ mod sent_notes_to_internal; mod shardtree_support; mod ufvk_support; mod utxos_table; +mod v_sapling_shard_unscanned_ranges; mod v_transactions_net; use schemer_rusqlite::RusqliteMigration; @@ -36,6 +37,8 @@ pub(super) fn all_migrations( // received_notes_nullable_nf // / | \ // shardtree_support nullifier_map sapling_memo_consistency + // | + // v_sapling_shard_unscanned_ranges vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -58,5 +61,8 @@ pub(super) fn all_migrations( Box::new(sapling_memo_consistency::Migration { params: params.clone(), }), + Box::new(v_sapling_shard_unscanned_ranges::Migration { + params: params.clone(), + }), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs new file mode 100644 index 0000000000..06dab783ea --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs @@ -0,0 +1,75 @@ +//! This migration adds a view that returns the un-scanned ranges associated with each sapling note +//! commitment tree shard. + +use std::collections::HashSet; + +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; +use zcash_client_backend::data_api::scanning::ScanPriority; +use zcash_primitives::consensus; + +use crate::wallet::{init::WalletMigrationError, scanning::priority_code}; + +use super::shardtree_support; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xfa934bdc_97b6_4980_8a83_b2cb1ac465fd); + +pub(super) struct Migration

{ + pub(super) params: P, +} + +impl

schemer::Migration for Migration

{ + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [shardtree_support::MIGRATION_ID].into_iter().collect() + } + + fn description(&self) -> &'static str { + "Adds a view that returns the un-scanned ranges associated with each sapling note commitment tree shard." + } +} + +impl RusqliteMigration for Migration

{ + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + transaction.execute_batch( + &format!( + "CREATE VIEW v_sapling_shard_unscanned_ranges AS + SELECT + shard.shard_index, + shard.shard_index << 16 AS start_position, + (shard.shard_index + 1) << 16 AS end_position_exclusive, + IFNULL(prev_shard.subtree_end_height, {}) AS subtree_start_height, + shard.subtree_end_height AS subtree_end_height, + shard.contains_marked, + scan_queue.block_range_start, + scan_queue.block_range_end, + scan_queue.priority + FROM sapling_tree_shards shard + LEFT OUTER JOIN sapling_tree_shards prev_shard + ON shard.shard_index = prev_shard.shard_index + 1 + INNER JOIN scan_queue ON + (scan_queue.block_range_start BETWEEN subtree_start_height AND shard.subtree_end_height) OR + ((scan_queue.block_range_end - 1) BETWEEN subtree_start_height AND shard.subtree_end_height) OR + ( + scan_queue.block_range_start <= prev_shard.subtree_end_height + AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height + ) + WHERE scan_queue.priority != {}", + u32::from(self.params.activation_height(consensus::NetworkUpgrade::Sapling).unwrap()), + priority_code(&ScanPriority::Scanned), + ) + )?; + + Ok(()) + } + + fn down(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + transaction.execute_batch("DROP VIEW v_sapling_shard_unscanned_ranges;")?; + Ok(()) + } +} diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 2d0499b3e9..b8891efe47 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -139,7 +139,15 @@ pub(crate) fn get_spendable_sapling_notes( WHERE account = :account AND spent IS NULL AND transactions.block <= :anchor_height - AND id_note NOT IN rarray(:exclude)", + AND id_note NOT IN rarray(:exclude) + AND NOT EXISTS ( + SELECT 1 FROM v_sapling_shard_unscanned_ranges unscanned + -- select all the unscanned ranges involving the shard containing this note + WHERE sapling_received_notes.commitment_tree_position >= unscanned.start_position + AND sapling_received_notes.commitment_tree_position < unscanned.end_position_exclusive + -- exclude unscanned ranges above the anchor height which don't affect spendability + AND unscanned.block_range_start <= :anchor_height + )", )?; let excluded: Vec = exclude.iter().map(|n| Value::from(n.0)).collect(); @@ -165,9 +173,7 @@ pub(crate) fn select_spendable_sapling_notes( exclude: &[ReceivedNoteId], ) -> Result>, SqliteClientError> { // The goal of this SQL statement is to select the oldest notes until the required - // value has been reached, and then fetch the witnesses at the desired height for the - // selected notes. This is achieved in several steps: - // + // value has been reached. // 1) Use a window function to create a view of all notes, ordered from oldest to // newest, with an additional column containing a running sum: // - Unspent notes accumulate the values of all unspent notes in that note's @@ -188,11 +194,20 @@ pub(crate) fn select_spendable_sapling_notes( SUM(value) OVER (PARTITION BY account, spent ORDER BY id_note) AS so_far FROM sapling_received_notes - INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx + INNER JOIN transactions + ON transactions.id_tx = sapling_received_notes.tx WHERE account = :account AND spent IS NULL AND transactions.block <= :anchor_height AND id_note NOT IN rarray(:exclude) + AND NOT EXISTS ( + SELECT 1 FROM v_sapling_shard_unscanned_ranges unscanned + -- select all the unscanned ranges involving the shard containing this note + WHERE sapling_received_notes.commitment_tree_position >= unscanned.start_position + AND sapling_received_notes.commitment_tree_position < unscanned.end_position_exclusive + -- exclude unscanned ranges above the anchor height which don't affect spendability + AND unscanned.block_range_start <= :anchor_height + ) ) SELECT id_note, diversifier, value, rcm, commitment_tree_position FROM eligible WHERE so_far < :target_value @@ -370,7 +385,7 @@ pub(crate) mod tests { block::BlockHash, consensus::BranchId, legacy::TransparentAddress, - memo::Memo, + memo::{Memo, MemoBytes}, sapling::{ note_encryption::try_sapling_output_recovery, prover::TxProver, Note, PaymentAddress, }, @@ -418,10 +433,7 @@ pub(crate) mod tests { zcash_client_backend::{ data_api::wallet::shield_transparent_funds, wallet::WalletTransparentOutput, }, - zcash_primitives::{ - memo::MemoBytes, - transaction::components::{amount::NonNegativeAmount, OutPoint, TxOut}, - }, + zcash_primitives::transaction::components::{amount::NonNegativeAmount, OutPoint, TxOut}, }; pub(crate) fn test_prover() -> impl TxProver { @@ -555,8 +567,8 @@ pub(crate) mod tests { let mut stmt_sent_notes = db_data .conn .prepare( - "SELECT output_index - FROM sent_notes + "SELECT output_index + FROM sent_notes JOIN transactions ON transactions.id_tx = sent_notes.tx WHERE transactions.txid = ?", ) From 0e7ee0ebd7a1a1eedfa3625d568448ccd19055a0 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 9 Aug 2023 15:51:02 -0600 Subject: [PATCH 0102/1122] zcash_client_sqlite: Ensure that the shard at the chain tip is complete to anchor height. If the tip shard has any un-scanned range below the anchor height, we can't compute witnesses for any of our notes. --- zcash_client_sqlite/src/wallet/sapling.rs | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index b8891efe47..425e0d12d3 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -132,6 +132,19 @@ pub(crate) fn get_spendable_sapling_notes( anchor_height: BlockHeight, exclude: &[ReceivedNoteId], ) -> Result>, SqliteClientError> { + let mut stmt_unscanned_tip = conn.prepare_cached( + "SELECT 1 FROM v_sapling_shard_unscanned_ranges + WHERE :anchor_height BETWEEN subtree_start_height AND IFNULL(subtree_end_height, :anchor_height) + AND block_range_start <= :anchor_height", + )?; + let mut unscanned = + stmt_unscanned_tip.query(named_params![":anchor_height": &u32::from(anchor_height),])?; + if unscanned.next()?.is_some() { + // if the tip shard has unscanned ranges below the anchor height, none of our notes can be + // spent + return Ok(vec![]); + } + let mut stmt_select_notes = conn.prepare_cached( "SELECT id_note, diversifier, value, rcm, commitment_tree_position FROM sapling_received_notes @@ -172,6 +185,19 @@ pub(crate) fn select_spendable_sapling_notes( anchor_height: BlockHeight, exclude: &[ReceivedNoteId], ) -> Result>, SqliteClientError> { + let mut stmt_unscanned_tip = conn.prepare_cached( + "SELECT 1 FROM v_sapling_shard_unscanned_ranges + WHERE :anchor_height BETWEEN subtree_start_height AND IFNULL(subtree_end_height, :anchor_height) + AND block_range_start <= :anchor_height", + )?; + let mut unscanned = + stmt_unscanned_tip.query(named_params![":anchor_height": &u32::from(anchor_height),])?; + if unscanned.next()?.is_some() { + // if the tip shard has unscanned ranges below the anchor height, none of our notes can be + // spent + return Ok(vec![]); + } + // The goal of this SQL statement is to select the oldest notes until the required // value has been reached. // 1) Use a window function to create a view of all notes, ordered from oldest to From 75ca045786f9346b1d3a072991a8fba2d3a82b89 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 11 Aug 2023 14:42:03 -0600 Subject: [PATCH 0103/1122] zcash_client_sqlite: Note selection requires commitment tree positions to be known. Fixes #895 --- zcash_client_sqlite/src/wallet/sapling.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 425e0d12d3..9e1ed2e024 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -150,6 +150,7 @@ pub(crate) fn get_spendable_sapling_notes( FROM sapling_received_notes INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx WHERE account = :account + AND commitment_tree_position IS NOT NULL AND spent IS NULL AND transactions.block <= :anchor_height AND id_note NOT IN rarray(:exclude) @@ -223,6 +224,7 @@ pub(crate) fn select_spendable_sapling_notes( INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx WHERE account = :account + AND commitment_tree_position IS NOT NULL AND spent IS NULL AND transactions.block <= :anchor_height AND id_note NOT IN rarray(:exclude) From 1f8108e8dabb95b218952eb33f31d38a1441de94 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 16 Aug 2023 12:45:21 -0600 Subject: [PATCH 0104/1122] zcash_client_sqlite: Use constant for SAPLING_SHARD_HEIGHT in v_sapling_shard_unscanned_ranges --- .../init/migrations/v_sapling_shard_unscanned_ranges.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs index 06dab783ea..a038057d01 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs @@ -5,7 +5,7 @@ use std::collections::HashSet; use schemer_rusqlite::RusqliteMigration; use uuid::Uuid; -use zcash_client_backend::data_api::scanning::ScanPriority; +use zcash_client_backend::data_api::{scanning::ScanPriority, SAPLING_SHARD_HEIGHT}; use zcash_primitives::consensus; use crate::wallet::{init::WalletMigrationError, scanning::priority_code}; @@ -41,8 +41,8 @@ impl RusqliteMigration for Migration

{ "CREATE VIEW v_sapling_shard_unscanned_ranges AS SELECT shard.shard_index, - shard.shard_index << 16 AS start_position, - (shard.shard_index + 1) << 16 AS end_position_exclusive, + shard.shard_index << {} AS start_position, + (shard.shard_index + 1) << {} AS end_position_exclusive, IFNULL(prev_shard.subtree_end_height, {}) AS subtree_start_height, shard.subtree_end_height AS subtree_end_height, shard.contains_marked, @@ -60,6 +60,8 @@ impl RusqliteMigration for Migration

{ AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height ) WHERE scan_queue.priority != {}", + SAPLING_SHARD_HEIGHT, + SAPLING_SHARD_HEIGHT, u32::from(self.params.activation_height(consensus::NetworkUpgrade::Sapling).unwrap()), priority_code(&ScanPriority::Scanned), ) From b3be0318c69d129599f7bf94845251cc5d58f9a3 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 18 Aug 2023 22:08:37 -0600 Subject: [PATCH 0105/1122] zcash_client_sqlite: Initialize the scan queue as part of `init_blocks_table` Fixes #902 Fixes #898 --- zcash_client_backend/src/data_api/scanning.rs | 4 +- zcash_client_sqlite/src/wallet/init.rs | 44 +++++-- .../src/wallet/init/migrations.rs | 4 +- .../init/migrations/shardtree_support.rs | 24 ++-- .../v_sapling_shard_unscanned_ranges.rs | 2 +- zcash_client_sqlite/src/wallet/scanning.rs | 111 +++++++++++++++++- 6 files changed, 164 insertions(+), 25 deletions(-) diff --git a/zcash_client_backend/src/data_api/scanning.rs b/zcash_client_backend/src/data_api/scanning.rs index bb91ba35a5..dcd9e0ff69 100644 --- a/zcash_client_backend/src/data_api/scanning.rs +++ b/zcash_client_backend/src/data_api/scanning.rs @@ -6,7 +6,9 @@ use zcash_primitives::consensus::BlockHeight; /// Scanning range priority levels. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum ScanPriority { - /// Block ranges that have already been scanned have lowest priority. + /// Block ranges that are ignored have lowest priority. + Ignored, + /// Block ranges that have already been scanned will not be re-scanned. Scanned, /// Block ranges to be scanned to advance the fully-scanned height. Historic, diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index da9657ca45..a829ba5e0f 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -4,7 +4,7 @@ use incrementalmerkletree::Retention; use std::{collections::HashMap, fmt}; use tracing::debug; -use rusqlite::{self, types::ToSql}; +use rusqlite::{self, named_params}; use schemer::{Migrator, MigratorError}; use schemer_rusqlite::RusqliteAdapter; use secrecy::SecretVec; @@ -13,18 +13,27 @@ use uuid::Uuid; use zcash_primitives::{ block::BlockHash, - consensus::{self, BlockHeight}, + consensus::{self, BlockHeight, NetworkUpgrade}, merkle_tree::read_commitment_tree, sapling, transaction::components::amount::BalanceError, zip32::AccountId, }; -use zcash_client_backend::{data_api::SAPLING_SHARD_HEIGHT, keys::UnifiedFullViewingKey}; +use zcash_client_backend::{ + data_api::{ + scanning::{ScanPriority, ScanRange}, + SAPLING_SHARD_HEIGHT, + }, + keys::UnifiedFullViewingKey, +}; use crate::{error::SqliteClientError, wallet, WalletDb, PRUNING_DEPTH, SAPLING_TABLES_PREFIX}; -use super::commitment_tree::{self, SqliteShardStore}; +use super::{ + commitment_tree::{self, SqliteShardStore}, + scanning::insert_queue_entries, +}; mod migrations; @@ -309,15 +318,28 @@ pub fn init_blocks_table( wdb.conn.0.execute( "INSERT INTO blocks (height, hash, time, sapling_tree) - VALUES (?, ?, ?, ?)", - [ - u32::from(height).to_sql()?, - hash.0.to_sql()?, - time.to_sql()?, - sapling_tree.to_sql()?, + VALUES (:height, :hash, :time, :sapling_tree)", + named_params![ + ":height": u32::from(height), + ":hash": hash.0, + ":time": time, + ":sapling_tree": sapling_tree, ], )?; + if let Some(sapling_activation) = wdb.params.activation_height(NetworkUpgrade::Sapling) { + let scan_range_start = std::cmp::min(sapling_activation, height); + let scan_range_end = height + 1; + insert_queue_entries( + wdb.conn.0, + Some(ScanRange::from_parts( + scan_range_start..scan_range_end, + ScanPriority::Ignored, + )) + .iter(), + )?; + } + if let Some(nonempty_frontier) = block_end_tree.to_frontier().value() { debug!("Inserting frontier into ShardTree: {:?}", nonempty_frontier); let shard_store = @@ -582,7 +604,7 @@ mod tests { scan_queue.block_range_start <= prev_shard.subtree_end_height AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height ) - WHERE scan_queue.priority != {}", + WHERE scan_queue.priority > {}", u32::from(tests::network().activation_height(NetworkUpgrade::Sapling).unwrap()), priority_code(&ScanPriority::Scanned), ), diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 834960ec49..6fbaea7e20 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -56,7 +56,9 @@ pub(super) fn all_migrations( Box::new(add_transaction_views::Migration), Box::new(v_transactions_net::Migration), Box::new(received_notes_nullable_nf::Migration), - Box::new(shardtree_support::Migration), + Box::new(shardtree_support::Migration { + params: params.clone(), + }), Box::new(nullifier_map::Migration), Box::new(sapling_memo_consistency::Migration { params: params.clone(), diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 64e1c42fc8..e35d01805e 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -17,7 +17,7 @@ use zcash_client_backend::data_api::{ SAPLING_SHARD_HEIGHT, }; use zcash_primitives::{ - consensus::BlockHeight, + consensus::{self, BlockHeight, NetworkUpgrade}, merkle_tree::{read_commitment_tree, read_incremental_witness}, sapling, }; @@ -39,9 +39,11 @@ pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( b"\x8b\xe5\xf5\x12\xbc\xce\x6c\xbf", ); -pub(super) struct Migration; +pub(super) struct Migration

{ + pub(super) params: P, +} -impl schemer::Migration for Migration { +impl

schemer::Migration for Migration

{ fn id(&self) -> Uuid { MIGRATION_ID } @@ -57,7 +59,7 @@ impl schemer::Migration for Migration { } } -impl RusqliteMigration for Migration { +impl RusqliteMigration for Migration

{ type Error = WalletMigrationError; fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { @@ -262,13 +264,17 @@ impl RusqliteMigration for Migration { if let Some((start, end)) = block_height_extrema { // `ScanRange` uses an exclusive upper bound. let chain_end = end + 1; + let ignored_range = + self.params + .activation_height(NetworkUpgrade::Sapling) + .map(|sapling_activation| { + let ignored_range_start = std::cmp::min(sapling_activation, start); + ScanRange::from_parts(ignored_range_start..start, ScanPriority::Ignored) + }); + let scanned_range = ScanRange::from_parts(start..chain_end, ScanPriority::Scanned); insert_queue_entries( transaction, - Some(ScanRange::from_parts( - start..chain_end, - ScanPriority::Scanned, - )) - .iter(), + ignored_range.iter().chain(Some(scanned_range).iter()), )?; } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs index a038057d01..770ecddc5e 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs @@ -59,7 +59,7 @@ impl RusqliteMigration for Migration

{ scan_queue.block_range_start <= prev_shard.subtree_end_height AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height ) - WHERE scan_queue.priority != {}", + WHERE scan_queue.priority > {}", SAPLING_SHARD_HEIGHT, SAPLING_SHARD_HEIGHT, u32::from(self.params.activation_height(consensus::NetworkUpgrade::Sapling).unwrap()), diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 83781837ac..646f2b361c 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -52,6 +52,7 @@ impl From for Dominance { pub(crate) fn parse_priority_code(code: i64) -> Option { use ScanPriority::*; match code { + 0 => Some(Ignored), 10 => Some(Scanned), 20 => Some(Historic), 30 => Some(OpenAdjacent), @@ -65,6 +66,7 @@ pub(crate) fn parse_priority_code(code: i64) -> Option { pub(crate) fn priority_code(priority: &ScanPriority) -> i64 { use ScanPriority::*; match priority { + Ignored => 0, Scanned => 10, Historic => 20, OpenAdjacent => 30, @@ -745,7 +747,10 @@ mod tests { WalletCommitmentTrees, WalletRead, WalletWrite, }; use zcash_primitives::{ - block::BlockHash, consensus::BlockHeight, sapling::Node, transaction::components::Amount, + block::BlockHash, + consensus::{BlockHeight, NetworkUpgrade, Parameters}, + sapling::Node, + transaction::components::Amount, }; use crate::{ @@ -754,7 +759,10 @@ mod tests { self, fake_compact_block, init_test_accounts_table, insert_into_cache, sapling_activation_height, AddressType, }, - wallet::{init::init_wallet_db, scanning::suggest_scan_ranges}, + wallet::{ + init::{init_blocks_table, init_wallet_db}, + scanning::suggest_scan_ranges, + }, BlockDb, WalletDb, }; @@ -1212,4 +1220,103 @@ mod tests { ] ); } + + #[test] + fn init_blocks_table_creates_ignored_range() { + use ScanPriority::*; + + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); + + let sap_active = db_data + .params + .activation_height(NetworkUpgrade::Sapling) + .unwrap(); + // Initialise the blocks table. We use Canopy activation as an arbitrary birthday height + // that's greater than Sapling activation. + let birthday_height = db_data + .params + .activation_height(NetworkUpgrade::Canopy) + .unwrap(); + init_blocks_table( + &mut db_data, + birthday_height, + BlockHash([1; 32]), + 1, + &[0x0, 0x0, 0x0], + ) + .unwrap(); + + let expected = vec![ + // The range below the wallet's birthday height is ignored + scan_range( + u32::from(sap_active)..u32::from(birthday_height + 1), + Ignored, + ), + ]; + assert_matches!( + suggest_scan_ranges(&db_data.conn, Ignored), + Ok(scan_ranges) if scan_ranges == expected + ); + + // Set up some shard history + db_data + .put_sapling_subtree_roots( + 0, + &[ + // Add the end of a commitment tree below the wallet birthday. We currently + // need to scan from this height up to the tip to make notes spendable, though + // this should not be necessary as we have added a frontier that should + // complete the left-hand side of the required shard; this can be fixed once we + // have proper account birthdays. + CommitmentTreeRoot::from_parts( + birthday_height - 1000, + // fake a hash, the value doesn't matter + Node::empty_leaf(), + ), + ], + ) + .unwrap(); + + // Update the chain tip + let tip_height = db_data + .params + .activation_height(NetworkUpgrade::Nu5) + .unwrap(); + db_data.update_chain_tip(tip_height).unwrap(); + + // Verify that the suggested scan ranges match what is expected + let expected = vec![ + // The birthday height was "last scanned" (as the wallet birthday) so we verify 10 + // blocks starting at that height. + scan_range( + u32::from(birthday_height)..u32::from(birthday_height + 10), + Verify, + ), + // The remainder of the shard after the verify segment is required in order to make + // notes spendable, so it has priority `ChainTip` + scan_range( + u32::from(birthday_height + 10)..u32::from(tip_height + 1), + ChainTip, + ), + // The remainder of the shard prior to the birthday height must be scanned because the + // wallet doesn't know that it already has enough data from the initial frontier to + // avoid having to scan this range. + scan_range( + u32::from(birthday_height - 1000)..u32::from(birthday_height), + ChainTip, + ), + // The range below the wallet's birthday height is ignored + scan_range( + u32::from(sap_active)..u32::from(birthday_height - 1000), + Ignored, + ), + ]; + + assert_matches!( + suggest_scan_ranges(&db_data.conn, Ignored), + Ok(scan_ranges) if scan_ranges == expected + ); + } } From 61ac79c006ee6fb4b6051f444974251c0a6f9fce Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 21 Aug 2023 10:51:54 -0600 Subject: [PATCH 0106/1122] Add debug logging of ignored block range in `init_blocks_table` --- zcash_client_sqlite/src/wallet/init.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index a829ba5e0f..1a7f24cc9f 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -330,6 +330,10 @@ pub fn init_blocks_table( if let Some(sapling_activation) = wdb.params.activation_height(NetworkUpgrade::Sapling) { let scan_range_start = std::cmp::min(sapling_activation, height); let scan_range_end = height + 1; + debug!( + "Setting ignored block range {}..{}", + scan_range_start, scan_range_end + ); insert_queue_entries( wdb.conn.0, Some(ScanRange::from_parts( From 4d6e236e23869ccaa59738c250a8e741fb90549f Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 21 Aug 2023 17:13:45 -0600 Subject: [PATCH 0107/1122] zcash_client_sqlite: minor test documentation improvement. Co-authored-by: Daira Emma Hopwood --- zcash_client_sqlite/src/wallet/scanning.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 646f2b361c..719d6d082e 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -1249,7 +1249,7 @@ mod tests { .unwrap(); let expected = vec![ - // The range below the wallet's birthday height is ignored + // The range up to and including the wallet's birthday height is ignored. scan_range( u32::from(sap_active)..u32::from(birthday_height + 1), Ignored, From 9859066c727cf52cff5803f3cd958883fb5fc90c Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 24 Aug 2023 07:47:30 -0600 Subject: [PATCH 0108/1122] Fix API doc comment on UnifiedSpendingKey This fixes a simple copy/paste error of the docs from the unified viewing key. --- zcash_client_backend/src/keys.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_backend/src/keys.rs b/zcash_client_backend/src/keys.rs index d7480c74d6..285e65bb15 100644 --- a/zcash_client_backend/src/keys.rs +++ b/zcash_client_backend/src/keys.rs @@ -133,7 +133,7 @@ impl Era { } } -/// A set of viewing keys that are all associated with a single +/// A set of spending keys that are all associated with a single /// ZIP-0032 account identifier. #[derive(Clone, Debug)] #[doc(hidden)] From 7ba36c7bd2f2588fb96f3ddd1079177c5387e873 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 3 Aug 2023 17:28:09 +0000 Subject: [PATCH 0109/1122] zcash_client_sqlite: Move testing helper functions into module --- zcash_client_sqlite/src/chain.rs | 48 +-- zcash_client_sqlite/src/lib.rs | 319 ++---------------- zcash_client_sqlite/src/testing.rs | 304 +++++++++++++++++ zcash_client_sqlite/src/wallet.rs | 8 +- .../src/wallet/commitment_tree.rs | 6 +- zcash_client_sqlite/src/wallet/init.rs | 41 +-- .../init/migrations/add_transaction_views.rs | 16 +- .../migrations/received_notes_nullable_nf.rs | 8 +- .../init/migrations/v_transactions_net.rs | 12 +- zcash_client_sqlite/src/wallet/sapling.rs | 84 ++--- zcash_client_sqlite/src/wallet/scanning.rs | 8 +- 11 files changed, 440 insertions(+), 414 deletions(-) create mode 100644 zcash_client_sqlite/src/testing.rs diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index f95f6b63ad..e23321eb73 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -348,7 +348,7 @@ mod tests { use crate::{ chain::init::init_cache_database, - tests::{ + testing::{ self, fake_compact_block, fake_compact_block_spending, init_test_accounts_table, insert_into_cache, sapling_activation_height, AddressType, }, @@ -363,7 +363,7 @@ mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet @@ -386,7 +386,7 @@ mod tests { // Scan the cache scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -409,7 +409,7 @@ mod tests { // Scanning should detect no inconsistencies assert_matches!( scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height() + 1, @@ -426,7 +426,7 @@ mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet @@ -455,7 +455,7 @@ mod tests { // Scanning the cache should find no inconsistencies assert_matches!( scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -487,7 +487,7 @@ mod tests { // Data+cache chain should be invalid at the data/cache boundary assert_matches!( scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height() + 2, @@ -505,7 +505,7 @@ mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet @@ -542,7 +542,7 @@ mod tests { // Scan the cache scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -584,7 +584,7 @@ mod tests { // Scan the cache again scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -606,7 +606,7 @@ mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet @@ -626,7 +626,7 @@ mod tests { ); insert_into_cache(&db_cache, &cb1); scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -660,7 +660,7 @@ mod tests { insert_into_cache(&db_cache, &cb3); assert_matches!( scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height() + 2, @@ -672,7 +672,7 @@ mod tests { // If we add a block of height SAPLING_ACTIVATION_HEIGHT + 1, we can now scan that insert_into_cache(&db_cache, &cb2); scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height() + 1, @@ -701,7 +701,7 @@ mod tests { assert_matches!( spend( &mut db_data, - &tests::network(), + &testing::network(), crate::wallet::sapling::tests::test_prover(), &input_selector, &usk, @@ -720,7 +720,7 @@ mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet @@ -746,7 +746,7 @@ mod tests { // Scan the cache scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -774,7 +774,7 @@ mod tests { // Scan the cache again scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height() + 1, @@ -796,7 +796,7 @@ mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet @@ -822,7 +822,7 @@ mod tests { // Scan the cache scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -855,7 +855,7 @@ mod tests { // Scan the cache again scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height() + 1, @@ -877,7 +877,7 @@ mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet @@ -920,7 +920,7 @@ mod tests { // Scan the spending block first. scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height() + 1, @@ -936,7 +936,7 @@ mod tests { // Now scan the block in which we received the note that was spent. scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 4935e2c5c9..df4f40785e 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -90,6 +90,9 @@ pub mod serialization; pub mod wallet; use wallet::commitment_tree::{self, put_shard_roots}; +#[cfg(test)] +mod testing; + /// The maximum number of blocks the wallet is allowed to rewind. This is /// consistent with the bound in zcashd, and allows block data deeper than /// this delta from the chain tip to be pruned. @@ -1082,317 +1085,33 @@ extern crate assert_matches; #[cfg(test)] mod tests { - use prost::Message; - use rand_core::{OsRng, RngCore}; - use rusqlite::params; - use std::collections::HashMap; + use zcash_client_backend::data_api::{WalletRead, WalletWrite}; - #[cfg(feature = "unstable")] - use std::{fs::File, path::Path}; - - #[cfg(feature = "transparent-inputs")] - use zcash_primitives::{legacy, legacy::keys::IncomingViewingKey}; + use crate::{ + testing::{init_test_accounts_table_ufvk, network}, + wallet::init::init_wallet_db, + AccountId, WalletDb, + }; - use zcash_note_encryption::Domain; + #[cfg(feature = "unstable")] use zcash_primitives::{ block::BlockHash, - consensus::{BlockHeight, Network, NetworkUpgrade, Parameters}, - legacy::TransparentAddress, - memo::MemoBytes, - sapling::{ - note_encryption::{sapling_note_encryption, SaplingDomain}, - util::generate_random_rseed, - value::NoteValue, - Note, Nullifier, PaymentAddress, - }, + consensus::{BlockHeight, Parameters}, transaction::components::Amount, - zip32::{sapling::DiversifiableFullViewingKey, DiversifierIndex}, }; - use zcash_client_backend::{ - data_api::{WalletRead, WalletWrite}, - keys::{sapling, UnifiedFullViewingKey}, - proto::compact_formats::{ - self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, - }, - }; - - use crate::{ - wallet::init::{init_accounts_table, init_wallet_db}, - AccountId, WalletDb, - }; - - use super::BlockDb; + #[cfg(feature = "unstable")] + use zcash_client_backend::keys::sapling; #[cfg(feature = "unstable")] - use super::{ - chain::{init::init_blockmeta_db, BlockMeta}, + use crate::{ + chain::init::init_blockmeta_db, + testing::{fake_compact_block, store_in_fsblockdb, AddressType}, FsBlockDb, }; - #[cfg(feature = "mainnet")] - pub(crate) fn network() -> Network { - Network::MainNetwork - } - - #[cfg(not(feature = "mainnet"))] - pub(crate) fn network() -> Network { - Network::TestNetwork - } - - #[cfg(feature = "mainnet")] - pub(crate) fn sapling_activation_height() -> BlockHeight { - Network::MainNetwork - .activation_height(NetworkUpgrade::Sapling) - .unwrap() - } - - #[cfg(not(feature = "mainnet"))] - pub(crate) fn sapling_activation_height() -> BlockHeight { - Network::TestNetwork - .activation_height(NetworkUpgrade::Sapling) - .unwrap() - } - - #[cfg(test)] - pub(crate) fn init_test_accounts_table( - db_data: &mut WalletDb, - ) -> (DiversifiableFullViewingKey, Option) { - let (ufvk, taddr) = init_test_accounts_table_ufvk(db_data); - (ufvk.sapling().unwrap().clone(), taddr) - } - - #[cfg(test)] - pub(crate) fn init_test_accounts_table_ufvk( - db_data: &mut WalletDb, - ) -> (UnifiedFullViewingKey, Option) { - let seed = [0u8; 32]; - let account = AccountId::from(0); - let extsk = sapling::spending_key(&seed, network().coin_type(), account); - let dfvk = extsk.to_diversifiable_full_viewing_key(); - - #[cfg(feature = "transparent-inputs")] - let (tkey, taddr) = { - let tkey = legacy::keys::AccountPrivKey::from_seed(&network(), &seed, account) - .unwrap() - .to_account_pubkey(); - let taddr = tkey.derive_external_ivk().unwrap().default_address().0; - (Some(tkey), Some(taddr)) - }; - - #[cfg(not(feature = "transparent-inputs"))] - let taddr = None; - - let ufvk = UnifiedFullViewingKey::new( - #[cfg(feature = "transparent-inputs")] - tkey, - Some(dfvk), - None, - ) - .unwrap(); - - let ufvks = HashMap::from([(account, ufvk.clone())]); - init_accounts_table(db_data, &ufvks).unwrap(); - - (ufvk, taddr) - } - - #[allow(dead_code)] - pub(crate) enum AddressType { - DefaultExternal, - DiversifiedExternal(DiversifierIndex), - Internal, - } - - /// Create a fake CompactBlock at the given height, containing a single output paying - /// an address. Returns the CompactBlock and the nullifier for the new note. - pub(crate) fn fake_compact_block( - height: BlockHeight, - prev_hash: BlockHash, - dfvk: &DiversifiableFullViewingKey, - req: AddressType, - value: Amount, - initial_sapling_tree_size: u32, - ) -> (CompactBlock, Nullifier) { - let to = match req { - AddressType::DefaultExternal => dfvk.default_address().1, - AddressType::DiversifiedExternal(idx) => dfvk.find_address(idx).unwrap().1, - AddressType::Internal => dfvk.change_address().1, - }; - - // Create a fake Note for the account - let mut rng = OsRng; - let rseed = generate_random_rseed(&network(), height, &mut rng); - let note = Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); - let encryptor = sapling_note_encryption::<_, Network>( - Some(dfvk.fvk().ovk), - note.clone(), - MemoBytes::empty(), - &mut rng, - ); - let cmu = note.cmu().to_bytes().to_vec(); - let ephemeral_key = SaplingDomain::::epk_bytes(encryptor.epk()) - .0 - .to_vec(); - let enc_ciphertext = encryptor.encrypt_note_plaintext(); - - // Create a fake CompactBlock containing the note - let cout = CompactSaplingOutput { - cmu, - ephemeral_key, - ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), - }; - let mut ctx = CompactTx::default(); - let mut txid = vec![0; 32]; - rng.fill_bytes(&mut txid); - ctx.hash = txid; - ctx.outputs.push(cout); - let mut cb = CompactBlock { - hash: { - let mut hash = vec![0; 32]; - rng.fill_bytes(&mut hash); - hash - }, - height: height.into(), - ..Default::default() - }; - cb.prev_hash.extend_from_slice(&prev_hash.0); - cb.vtx.push(ctx); - cb.chain_metadata = Some(compact::ChainMetadata { - sapling_commitment_tree_size: initial_sapling_tree_size - + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), - ..Default::default() - }); - (cb, note.nf(&dfvk.fvk().vk.nk, 0)) - } - - /// Create a fake CompactBlock at the given height, spending a single note from the - /// given address. - pub(crate) fn fake_compact_block_spending( - height: BlockHeight, - prev_hash: BlockHash, - (nf, in_value): (Nullifier, Amount), - dfvk: &DiversifiableFullViewingKey, - to: PaymentAddress, - value: Amount, - initial_sapling_tree_size: u32, - ) -> CompactBlock { - let mut rng = OsRng; - let rseed = generate_random_rseed(&network(), height, &mut rng); - - // Create a fake CompactBlock containing the note - let cspend = CompactSaplingSpend { nf: nf.to_vec() }; - let mut ctx = CompactTx::default(); - let mut txid = vec![0; 32]; - rng.fill_bytes(&mut txid); - ctx.hash = txid; - ctx.spends.push(cspend); - - // Create a fake Note for the payment - ctx.outputs.push({ - let note = Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); - let encryptor = sapling_note_encryption::<_, Network>( - Some(dfvk.fvk().ovk), - note.clone(), - MemoBytes::empty(), - &mut rng, - ); - let cmu = note.cmu().to_bytes().to_vec(); - let ephemeral_key = SaplingDomain::::epk_bytes(encryptor.epk()) - .0 - .to_vec(); - let enc_ciphertext = encryptor.encrypt_note_plaintext(); - - CompactSaplingOutput { - cmu, - ephemeral_key, - ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), - } - }); - - // Create a fake Note for the change - ctx.outputs.push({ - let change_addr = dfvk.default_address().1; - let rseed = generate_random_rseed(&network(), height, &mut rng); - let note = Note::from_parts( - change_addr, - NoteValue::from_raw((in_value - value).unwrap().into()), - rseed, - ); - let encryptor = sapling_note_encryption::<_, Network>( - Some(dfvk.fvk().ovk), - note.clone(), - MemoBytes::empty(), - &mut rng, - ); - let cmu = note.cmu().to_bytes().to_vec(); - let ephemeral_key = SaplingDomain::::epk_bytes(encryptor.epk()) - .0 - .to_vec(); - let enc_ciphertext = encryptor.encrypt_note_plaintext(); - - CompactSaplingOutput { - cmu, - ephemeral_key, - ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), - } - }); - - let mut cb = CompactBlock { - hash: { - let mut hash = vec![0; 32]; - rng.fill_bytes(&mut hash); - hash - }, - height: height.into(), - ..Default::default() - }; - cb.prev_hash.extend_from_slice(&prev_hash.0); - cb.vtx.push(ctx); - cb.chain_metadata = Some(compact::ChainMetadata { - sapling_commitment_tree_size: initial_sapling_tree_size - + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), - ..Default::default() - }); - cb - } - - /// Insert a fake CompactBlock into the cache DB. - pub(crate) fn insert_into_cache(db_cache: &BlockDb, cb: &CompactBlock) { - let cb_bytes = cb.encode_to_vec(); - db_cache - .0 - .prepare("INSERT INTO compactblocks (height, data) VALUES (?, ?)") - .unwrap() - .execute(params![u32::from(cb.height()), cb_bytes,]) - .unwrap(); - } - #[cfg(feature = "unstable")] - pub(crate) fn store_in_fsblockdb>( - fsblockdb_root: P, - cb: &CompactBlock, - ) -> BlockMeta { - use std::io::Write; - - let meta = BlockMeta { - height: cb.height(), - block_hash: cb.hash(), - block_time: cb.time, - sapling_outputs_count: cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum(), - orchard_actions_count: cb.vtx.iter().map(|tx| tx.actions.len() as u32).sum(), - }; - - let blocks_dir = fsblockdb_root.as_ref().join("blocks"); - let block_path = meta.block_file_path(&blocks_dir); - - File::create(block_path) - .unwrap() - .write_all(&cb.encode_to_vec()) - .unwrap(); - - meta - } + use super::BlockDb; #[test] pub(crate) fn get_next_available_address() { @@ -1422,7 +1141,9 @@ mod tests { use secrecy::Secret; use tempfile::NamedTempFile; - use crate::{chain::init::init_cache_database, wallet::init::init_wallet_db}; + use crate::{ + chain::init::init_cache_database, testing::network, wallet::init::init_wallet_db, + }; let cache_file = NamedTempFile::new().unwrap(); let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs new file mode 100644 index 0000000000..1dc76db02c --- /dev/null +++ b/zcash_client_sqlite/src/testing.rs @@ -0,0 +1,304 @@ +use std::collections::HashMap; + +#[cfg(feature = "unstable")] +use std::{fs::File, path::Path}; + +use prost::Message; +use rand_core::{OsRng, RngCore}; +use rusqlite::params; + +use zcash_client_backend::{ + keys::{sapling, UnifiedFullViewingKey}, + proto::compact_formats::{ + self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, + }, +}; +use zcash_note_encryption::Domain; +use zcash_primitives::{ + block::BlockHash, + consensus::{BlockHeight, Network, NetworkUpgrade, Parameters}, + legacy::TransparentAddress, + memo::MemoBytes, + sapling::{ + note_encryption::{sapling_note_encryption, SaplingDomain}, + util::generate_random_rseed, + value::NoteValue, + Note, Nullifier, PaymentAddress, + }, + transaction::components::Amount, + zip32::{sapling::DiversifiableFullViewingKey, DiversifierIndex}, +}; + +#[cfg(feature = "transparent-inputs")] +use zcash_primitives::{legacy, legacy::keys::IncomingViewingKey}; + +use crate::{wallet::init::init_accounts_table, AccountId, WalletDb}; + +use super::BlockDb; + +#[cfg(feature = "unstable")] +use super::chain::BlockMeta; + +#[cfg(feature = "mainnet")] +pub(crate) fn network() -> Network { + Network::MainNetwork +} + +#[cfg(not(feature = "mainnet"))] +pub(crate) fn network() -> Network { + Network::TestNetwork +} + +#[cfg(feature = "mainnet")] +pub(crate) fn sapling_activation_height() -> BlockHeight { + Network::MainNetwork + .activation_height(NetworkUpgrade::Sapling) + .unwrap() +} + +#[cfg(not(feature = "mainnet"))] +pub(crate) fn sapling_activation_height() -> BlockHeight { + Network::TestNetwork + .activation_height(NetworkUpgrade::Sapling) + .unwrap() +} + +#[cfg(test)] +pub(crate) fn init_test_accounts_table( + db_data: &mut WalletDb, +) -> (DiversifiableFullViewingKey, Option) { + let (ufvk, taddr) = init_test_accounts_table_ufvk(db_data); + (ufvk.sapling().unwrap().clone(), taddr) +} + +#[cfg(test)] +pub(crate) fn init_test_accounts_table_ufvk( + db_data: &mut WalletDb, +) -> (UnifiedFullViewingKey, Option) { + let seed = [0u8; 32]; + let account = AccountId::from(0); + let extsk = sapling::spending_key(&seed, network().coin_type(), account); + let dfvk = extsk.to_diversifiable_full_viewing_key(); + + #[cfg(feature = "transparent-inputs")] + let (tkey, taddr) = { + let tkey = legacy::keys::AccountPrivKey::from_seed(&network(), &seed, account) + .unwrap() + .to_account_pubkey(); + let taddr = tkey.derive_external_ivk().unwrap().default_address().0; + (Some(tkey), Some(taddr)) + }; + + #[cfg(not(feature = "transparent-inputs"))] + let taddr = None; + + let ufvk = UnifiedFullViewingKey::new( + #[cfg(feature = "transparent-inputs")] + tkey, + Some(dfvk), + None, + ) + .unwrap(); + + let ufvks = HashMap::from([(account, ufvk.clone())]); + init_accounts_table(db_data, &ufvks).unwrap(); + + (ufvk, taddr) +} + +#[allow(dead_code)] +pub(crate) enum AddressType { + DefaultExternal, + DiversifiedExternal(DiversifierIndex), + Internal, +} + +/// Create a fake CompactBlock at the given height, containing a single output paying +/// an address. Returns the CompactBlock and the nullifier for the new note. +pub(crate) fn fake_compact_block( + height: BlockHeight, + prev_hash: BlockHash, + dfvk: &DiversifiableFullViewingKey, + req: AddressType, + value: Amount, + initial_sapling_tree_size: u32, +) -> (CompactBlock, Nullifier) { + let to = match req { + AddressType::DefaultExternal => dfvk.default_address().1, + AddressType::DiversifiedExternal(idx) => dfvk.find_address(idx).unwrap().1, + AddressType::Internal => dfvk.change_address().1, + }; + + // Create a fake Note for the account + let mut rng = OsRng; + let rseed = generate_random_rseed(&network(), height, &mut rng); + let note = Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); + let encryptor = sapling_note_encryption::<_, Network>( + Some(dfvk.fvk().ovk), + note.clone(), + MemoBytes::empty(), + &mut rng, + ); + let cmu = note.cmu().to_bytes().to_vec(); + let ephemeral_key = SaplingDomain::::epk_bytes(encryptor.epk()) + .0 + .to_vec(); + let enc_ciphertext = encryptor.encrypt_note_plaintext(); + + // Create a fake CompactBlock containing the note + let cout = CompactSaplingOutput { + cmu, + ephemeral_key, + ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), + }; + let mut ctx = CompactTx::default(); + let mut txid = vec![0; 32]; + rng.fill_bytes(&mut txid); + ctx.hash = txid; + ctx.outputs.push(cout); + let mut cb = CompactBlock { + hash: { + let mut hash = vec![0; 32]; + rng.fill_bytes(&mut hash); + hash + }, + height: height.into(), + ..Default::default() + }; + cb.prev_hash.extend_from_slice(&prev_hash.0); + cb.vtx.push(ctx); + cb.chain_metadata = Some(compact::ChainMetadata { + sapling_commitment_tree_size: initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), + ..Default::default() + }); + (cb, note.nf(&dfvk.fvk().vk.nk, 0)) +} + +/// Create a fake CompactBlock at the given height, spending a single note from the +/// given address. +pub(crate) fn fake_compact_block_spending( + height: BlockHeight, + prev_hash: BlockHash, + (nf, in_value): (Nullifier, Amount), + dfvk: &DiversifiableFullViewingKey, + to: PaymentAddress, + value: Amount, + initial_sapling_tree_size: u32, +) -> CompactBlock { + let mut rng = OsRng; + let rseed = generate_random_rseed(&network(), height, &mut rng); + + // Create a fake CompactBlock containing the note + let cspend = CompactSaplingSpend { nf: nf.to_vec() }; + let mut ctx = CompactTx::default(); + let mut txid = vec![0; 32]; + rng.fill_bytes(&mut txid); + ctx.hash = txid; + ctx.spends.push(cspend); + + // Create a fake Note for the payment + ctx.outputs.push({ + let note = Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); + let encryptor = sapling_note_encryption::<_, Network>( + Some(dfvk.fvk().ovk), + note.clone(), + MemoBytes::empty(), + &mut rng, + ); + let cmu = note.cmu().to_bytes().to_vec(); + let ephemeral_key = SaplingDomain::::epk_bytes(encryptor.epk()) + .0 + .to_vec(); + let enc_ciphertext = encryptor.encrypt_note_plaintext(); + + CompactSaplingOutput { + cmu, + ephemeral_key, + ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), + } + }); + + // Create a fake Note for the change + ctx.outputs.push({ + let change_addr = dfvk.default_address().1; + let rseed = generate_random_rseed(&network(), height, &mut rng); + let note = Note::from_parts( + change_addr, + NoteValue::from_raw((in_value - value).unwrap().into()), + rseed, + ); + let encryptor = sapling_note_encryption::<_, Network>( + Some(dfvk.fvk().ovk), + note.clone(), + MemoBytes::empty(), + &mut rng, + ); + let cmu = note.cmu().to_bytes().to_vec(); + let ephemeral_key = SaplingDomain::::epk_bytes(encryptor.epk()) + .0 + .to_vec(); + let enc_ciphertext = encryptor.encrypt_note_plaintext(); + + CompactSaplingOutput { + cmu, + ephemeral_key, + ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), + } + }); + + let mut cb = CompactBlock { + hash: { + let mut hash = vec![0; 32]; + rng.fill_bytes(&mut hash); + hash + }, + height: height.into(), + ..Default::default() + }; + cb.prev_hash.extend_from_slice(&prev_hash.0); + cb.vtx.push(ctx); + cb.chain_metadata = Some(compact::ChainMetadata { + sapling_commitment_tree_size: initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), + ..Default::default() + }); + cb +} + +/// Insert a fake CompactBlock into the cache DB. +pub(crate) fn insert_into_cache(db_cache: &BlockDb, cb: &CompactBlock) { + let cb_bytes = cb.encode_to_vec(); + db_cache + .0 + .prepare("INSERT INTO compactblocks (height, data) VALUES (?, ?)") + .unwrap() + .execute(params![u32::from(cb.height()), cb_bytes,]) + .unwrap(); +} + +#[cfg(feature = "unstable")] +pub(crate) fn store_in_fsblockdb>( + fsblockdb_root: P, + cb: &CompactBlock, +) -> BlockMeta { + use std::io::Write; + + let meta = BlockMeta { + height: cb.height(), + block_hash: cb.hash(), + block_time: cb.time, + sapling_outputs_count: cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum(), + orchard_actions_count: cb.vtx.iter().map(|tx| tx.actions.len() as u32).sum(), + }; + + let blocks_dir = fsblockdb_root.as_ref().join("blocks"); + let block_path = meta.block_file_path(&blocks_dir); + + File::create(block_path) + .unwrap() + .write_all(&cb.encode_to_vec()) + .unwrap(); + + meta +} diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 9de950997f..8fb7ebe708 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1579,7 +1579,7 @@ mod tests { use zcash_client_backend::data_api::WalletRead; use crate::{ - tests, + testing, wallet::{get_current_address, init::init_wallet_db}, AccountId, WalletDb, }; @@ -1600,11 +1600,11 @@ mod tests { #[test] fn empty_database_has_no_balance() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet - tests::init_test_accounts_table(&mut db_data); + testing::init_test_accounts_table(&mut db_data); // The account should be empty assert_eq!( @@ -1635,7 +1635,7 @@ mod tests { #[cfg(feature = "transparent-inputs")] fn put_received_transparent_utxo() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index bce22ca7bc..3242080719 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -988,11 +988,11 @@ mod tests { use zcash_primitives::consensus::BlockHeight; use super::SqliteShardStore; - use crate::{tests, wallet::init::init_wallet_db, WalletDb, SAPLING_TABLES_PREFIX}; + use crate::{testing, wallet::init::init_wallet_db, WalletDb, SAPLING_TABLES_PREFIX}; fn new_tree(m: usize) -> ShardTree, 4, 3> { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); data_file.keep().unwrap(); init_wallet_db(&mut db_data, None).unwrap(); @@ -1040,7 +1040,7 @@ mod tests { #[test] fn put_shard_roots() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); data_file.keep().unwrap(); init_wallet_db(&mut db_data, None).unwrap(); diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 1a7f24cc9f..40913504c6 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -393,7 +393,7 @@ mod tests { use crate::{ error::SqliteClientError, - tests::{self, network}, + testing::{self, network}, wallet::scanning::priority_code, AccountId, WalletDb, }; @@ -415,7 +415,7 @@ mod tests { #[test] fn verify_schema() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); use regex::Regex; @@ -609,7 +609,7 @@ mod tests { AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height ) WHERE scan_queue.priority > {}", - u32::from(tests::network().activation_height(NetworkUpgrade::Sapling).unwrap()), + u32::from(testing::network().activation_height(NetworkUpgrade::Sapling).unwrap()), priority_code(&ScanPriority::Scanned), ), // v_transactions @@ -852,11 +852,11 @@ mod tests { )?; let address = encode_payment_address( - tests::network().hrp_sapling_payment_address(), + testing::network().hrp_sapling_payment_address(), &extfvk.default_address().1, ); let extfvk = encode_extended_full_viewing_key( - tests::network().hrp_sapling_extended_full_viewing_key(), + testing::network().hrp_sapling_extended_full_viewing_key(), extfvk, ); wdb.conn.execute( @@ -874,10 +874,10 @@ mod tests { let seed = [0xab; 32]; let account = AccountId::from(0); - let secret_key = sapling::spending_key(&seed, tests::network().coin_type(), account); + let secret_key = sapling::spending_key(&seed, testing::network().coin_type(), account); let extfvk = secret_key.to_extended_full_viewing_key(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_0_3_0(&mut db_data, &extfvk, account).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(seed.to_vec()))).unwrap(); } @@ -984,11 +984,11 @@ mod tests { )?; let address = encode_payment_address( - tests::network().hrp_sapling_payment_address(), + testing::network().hrp_sapling_payment_address(), &extfvk.default_address().1, ); let extfvk = encode_extended_full_viewing_key( - tests::network().hrp_sapling_extended_full_viewing_key(), + testing::network().hrp_sapling_extended_full_viewing_key(), extfvk, ); wdb.conn.execute( @@ -1040,10 +1040,10 @@ mod tests { let seed = [0xab; 32]; let account = AccountId::from(0); - let secret_key = sapling::spending_key(&seed, tests::network().coin_type(), account); + let secret_key = sapling::spending_key(&seed, testing::network().coin_type(), account); let extfvk = secret_key.to_extended_full_viewing_key(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_autoshielding(&mut db_data, &extfvk, account).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(seed.to_vec()))).unwrap(); } @@ -1150,9 +1150,9 @@ mod tests { [], )?; - let ufvk_str = ufvk.encode(&tests::network()); + let ufvk_str = ufvk.encode(&testing::network()); let address_str = - RecipientAddress::Unified(ufvk.default_address().0).encode(&tests::network()); + RecipientAddress::Unified(ufvk.default_address().0).encode(&testing::network()); wdb.conn.execute( "INSERT INTO accounts (account, ufvk, address, transparent_address) VALUES (?, ?, ?, '')", @@ -1168,7 +1168,7 @@ mod tests { { let taddr = RecipientAddress::Transparent(*ufvk.default_address().0.transparent().unwrap()) - .encode(&tests::network()); + .encode(&testing::network()); wdb.conn.execute( "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'000000')", [], @@ -1188,9 +1188,10 @@ mod tests { let seed = [0xab; 32]; let account = AccountId::from(0); - let secret_key = UnifiedSpendingKey::from_seed(&tests::network(), &seed, account).unwrap(); + let secret_key = + UnifiedSpendingKey::from_seed(&testing::network(), &seed, account).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_main( &mut db_data, &secret_key.to_unified_full_viewing_key(), @@ -1203,7 +1204,7 @@ mod tests { #[test] fn init_accounts_table_only_works_once() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // We can call the function as many times as we want with no data @@ -1272,7 +1273,7 @@ mod tests { #[test] fn init_blocks_table_only_works_once() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // First call with data should initialise the blocks table @@ -1299,14 +1300,14 @@ mod tests { #[test] fn init_accounts_table_stores_correct_address() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); let seed = [0u8; 32]; // Add an account to the wallet let account_id = AccountId::from(0); - let usk = UnifiedSpendingKey::from_seed(&tests::network(), &seed, account_id).unwrap(); + let usk = UnifiedSpendingKey::from_seed(&testing::network(), &seed, account_id).unwrap(); let ufvk = usk.to_unified_full_viewing_key(); let expected_address = ufvk.sapling().unwrap().default_address().1; let ufvks = HashMap::from([(account_id, ufvk)]); diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs index 06efb13277..0d217c545a 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs @@ -286,7 +286,7 @@ mod tests { use zcash_primitives::zip32::AccountId; use crate::{ - tests, + testing, wallet::init::{init_wallet_db_internal, migrations::addresses_table}, WalletDb, }; @@ -311,10 +311,10 @@ mod tests { #[test] fn transaction_views() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db_internal(&mut db_data, None, &[addresses_table::MIGRATION_ID]).unwrap(); let usk = - UnifiedSpendingKey::from_seed(&tests::network(), &[0u8; 32][..], AccountId::from(0)) + UnifiedSpendingKey::from_seed(&testing::network(), &[0u8; 32][..], AccountId::from(0)) .unwrap(); let ufvk = usk.to_unified_full_viewing_key(); @@ -322,7 +322,7 @@ mod tests { .conn .execute( "INSERT INTO accounts (account, ufvk) VALUES (0, ?)", - params![ufvk.encode(&tests::network())], + params![ufvk.encode(&testing::network())], ) .unwrap(); @@ -403,7 +403,7 @@ mod tests { #[cfg(feature = "transparent-inputs")] fn migrate_from_wm2() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db_internal( &mut db_data, None, @@ -440,7 +440,7 @@ mod tests { tx.write(&mut tx_bytes).unwrap(); let usk = - UnifiedSpendingKey::from_seed(&tests::network(), &[0u8; 32][..], AccountId::from(0)) + UnifiedSpendingKey::from_seed(&testing::network(), &[0u8; 32][..], AccountId::from(0)) .unwrap(); let ufvk = usk.to_unified_full_viewing_key(); let (ua, _) = ufvk.default_address(); @@ -451,11 +451,11 @@ mod tests { .ok() .map(|k| k.derive_address(0).unwrap()) }) - .map(|a| a.encode(&tests::network())); + .map(|a| a.encode(&testing::network())); db_data.conn.execute( "INSERT INTO accounts (account, ufvk, address, transparent_address) VALUES (0, ?, ?, ?)", - params![ufvk.encode(&tests::network()), ua.encode(&tests::network()), &taddr] + params![ufvk.encode(&testing::network()), ua.encode(&testing::network()), &taddr] ).unwrap(); db_data .conn diff --git a/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs b/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs index 5567d60dc9..2c946f3aa9 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs @@ -236,7 +236,7 @@ mod tests { use zcash_primitives::zip32::AccountId; use crate::{ - tests, + testing, wallet::init::{init_wallet_db_internal, migrations::v_transactions_net}, WalletDb, }; @@ -244,19 +244,19 @@ mod tests { #[test] fn received_notes_nullable_migration() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db_internal(&mut db_data, None, &[v_transactions_net::MIGRATION_ID]).unwrap(); // Create an account in the wallet let usk0 = - UnifiedSpendingKey::from_seed(&tests::network(), &[0u8; 32][..], AccountId::from(0)) + UnifiedSpendingKey::from_seed(&testing::network(), &[0u8; 32][..], AccountId::from(0)) .unwrap(); let ufvk0 = usk0.to_unified_full_viewing_key(); db_data .conn .execute( "INSERT INTO accounts (account, ufvk) VALUES (0, ?)", - params![ufvk0.encode(&tests::network())], + params![ufvk0.encode(&testing::network())], ) .unwrap(); diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs index 7f82cea4a4..b6bfe4a197 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs @@ -215,7 +215,7 @@ mod tests { use zcash_primitives::zip32::AccountId; use crate::{ - tests, + testing, wallet::init::{init_wallet_db_internal, migrations::add_transaction_views}, WalletDb, }; @@ -223,32 +223,32 @@ mod tests { #[test] fn v_transactions_net() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db_internal(&mut db_data, None, &[add_transaction_views::MIGRATION_ID]) .unwrap(); // Create two accounts in the wallet. let usk0 = - UnifiedSpendingKey::from_seed(&tests::network(), &[0u8; 32][..], AccountId::from(0)) + UnifiedSpendingKey::from_seed(&testing::network(), &[0u8; 32][..], AccountId::from(0)) .unwrap(); let ufvk0 = usk0.to_unified_full_viewing_key(); db_data .conn .execute( "INSERT INTO accounts (account, ufvk) VALUES (0, ?)", - params![ufvk0.encode(&tests::network())], + params![ufvk0.encode(&testing::network())], ) .unwrap(); let usk1 = - UnifiedSpendingKey::from_seed(&tests::network(), &[1u8; 32][..], AccountId::from(1)) + UnifiedSpendingKey::from_seed(&testing::network(), &[1u8; 32][..], AccountId::from(1)) .unwrap(); let ufvk1 = usk1.to_unified_full_viewing_key(); db_data .conn .execute( "INSERT INTO accounts (account, ufvk) VALUES (1, ?)", - params![ufvk1.encode(&tests::network())], + params![ufvk1.encode(&testing::network())], ) .unwrap(); diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 9e1ed2e024..0bfc60905b 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -448,7 +448,7 @@ pub(crate) mod tests { use crate::{ chain::init::init_cache_database, error::SqliteClientError, - tests::{ + testing::{ self, fake_compact_block, insert_into_cache, network, sapling_activation_height, AddressType, }, @@ -480,7 +480,7 @@ pub(crate) mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet @@ -500,7 +500,7 @@ pub(crate) mod tests { ); insert_into_cache(&db_cache, &cb); scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -540,7 +540,7 @@ pub(crate) mod tests { &GreedyInputSelector::new(change_strategy, DustOutputPolicy::default()); let proposal_result = propose_transfer::<_, _, _, Infallible>( &mut db_data, - &tests::network(), + &testing::network(), account, input_selector, request, @@ -551,7 +551,7 @@ pub(crate) mod tests { let change_memo = "Test change memo".parse::().unwrap(); let create_proposed_result = create_proposed_transaction::<_, _, Infallible, _>( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &usk, OvkPolicy::Sender, @@ -571,7 +571,7 @@ pub(crate) mod tests { .into_iter() .collect(); let decrypted_outputs = decrypt_transaction( - &tests::network(), + &testing::network(), sapling_activation_height() + 1, &tx, &ufvks, @@ -650,7 +650,7 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_on_incorrect_usk() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet @@ -667,7 +667,7 @@ pub(crate) mod tests { assert_matches!( create_spend_to_address( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &usk1, &to, @@ -683,7 +683,7 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_with_no_blocks() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet @@ -702,7 +702,7 @@ pub(crate) mod tests { assert_matches!( create_spend_to_address( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &usk, &to, @@ -722,7 +722,7 @@ pub(crate) mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet @@ -742,7 +742,7 @@ pub(crate) mod tests { ); insert_into_cache(&db_cache, &cb); scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -776,7 +776,7 @@ pub(crate) mod tests { .0; insert_into_cache(&db_cache, &cb); scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height() + 1, @@ -804,7 +804,7 @@ pub(crate) mod tests { assert_matches!( create_spend_to_address( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &usk, &to, @@ -836,7 +836,7 @@ pub(crate) mod tests { insert_into_cache(&db_cache, &cb); } scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height() + 2, @@ -848,7 +848,7 @@ pub(crate) mod tests { assert_matches!( create_spend_to_address( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &usk, &to, @@ -877,7 +877,7 @@ pub(crate) mod tests { .0; insert_into_cache(&db_cache, &cb); scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height() + 10, @@ -889,7 +889,7 @@ pub(crate) mod tests { assert_matches!( create_spend_to_address( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &usk, &to, @@ -909,7 +909,7 @@ pub(crate) mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet @@ -929,7 +929,7 @@ pub(crate) mod tests { ); insert_into_cache(&db_cache, &cb); scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -947,7 +947,7 @@ pub(crate) mod tests { assert_matches!( create_spend_to_address( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &usk, &to, @@ -963,7 +963,7 @@ pub(crate) mod tests { assert_matches!( create_spend_to_address( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &usk, &to, @@ -994,7 +994,7 @@ pub(crate) mod tests { insert_into_cache(&db_cache, &cb); } scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height() + 1, @@ -1006,7 +1006,7 @@ pub(crate) mod tests { assert_matches!( create_spend_to_address( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &usk, &to, @@ -1034,7 +1034,7 @@ pub(crate) mod tests { .0; insert_into_cache(&db_cache, &cb); scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height() + 42, @@ -1045,7 +1045,7 @@ pub(crate) mod tests { // Second spend should now succeed create_spend_to_address( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &usk, &to, @@ -1059,7 +1059,7 @@ pub(crate) mod tests { #[test] fn ovk_policy_prevents_recovery_from_chain() { - let network = tests::network(); + let network = testing::network(); let cache_file = NamedTempFile::new().unwrap(); let db_cache = BlockDb(Connection::open(cache_file.path()).unwrap()); init_cache_database(&db_cache).unwrap(); @@ -1085,7 +1085,7 @@ pub(crate) mod tests { ); insert_into_cache(&db_cache, &cb); scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -1116,7 +1116,7 @@ pub(crate) mod tests { > { let txid = create_spend_to_address( db_data, - &tests::network(), + &testing::network(), test_prover(), &usk, &to, @@ -1200,7 +1200,7 @@ pub(crate) mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet @@ -1220,7 +1220,7 @@ pub(crate) mod tests { ); insert_into_cache(&db_cache, &cb); scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -1246,7 +1246,7 @@ pub(crate) mod tests { assert_matches!( create_spend_to_address( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &usk, &to, @@ -1266,7 +1266,7 @@ pub(crate) mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet @@ -1286,7 +1286,7 @@ pub(crate) mod tests { ); insert_into_cache(&db_cache, &cb); scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -1312,7 +1312,7 @@ pub(crate) mod tests { assert_matches!( create_spend_to_address( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &usk, &to, @@ -1332,7 +1332,7 @@ pub(crate) mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet @@ -1366,7 +1366,7 @@ pub(crate) mod tests { } scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -1408,7 +1408,7 @@ pub(crate) mod tests { assert_matches!( spend( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &input_selector, &usk, @@ -1436,7 +1436,7 @@ pub(crate) mod tests { assert_matches!( spend( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &input_selector, &usk, @@ -1456,7 +1456,7 @@ pub(crate) mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); // Add an account to the wallet @@ -1495,7 +1495,7 @@ pub(crate) mod tests { ); insert_into_cache(&db_cache, &cb); scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, sapling_activation_height(), @@ -1506,7 +1506,7 @@ pub(crate) mod tests { assert_matches!( shield_transparent_funds( &mut db_data, - &tests::network(), + &testing::network(), test_prover(), &input_selector, NonNegativeAmount::from_u64(10000).unwrap(), diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 719d6d082e..27863ff9e5 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -755,7 +755,7 @@ mod tests { use crate::{ chain::init::init_cache_database, - tests::{ + testing::{ self, fake_compact_block, init_test_accounts_table, insert_into_cache, sapling_activation_height, AddressType, }, @@ -1098,7 +1098,7 @@ mod tests { init_cache_database(&db_cache).unwrap(); let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet. @@ -1159,7 +1159,7 @@ mod tests { assert_matches!( scan_cached_blocks( - &tests::network(), + &testing::network(), &db_cache, &mut db_data, initial_height, @@ -1226,7 +1226,7 @@ mod tests { use ScanPriority::*; let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); let sap_active = db_data From ce3e4cd34a5b6fbabaa87739de2950d168bd9dcc Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 28 Aug 2023 16:28:48 +0000 Subject: [PATCH 0110/1122] zcash_client_sqlite: Introduce `TestBuilder` and `TestRunner` These provide a standard way to set up tests, and helper methods for performing common test actions. --- zcash_client_sqlite/src/testing.rs | 592 +++++++++++++++++++++++++++-- 1 file changed, 555 insertions(+), 37 deletions(-) diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 1dc76db02c..5c0cbee8fe 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -1,17 +1,38 @@ -use std::collections::HashMap; +use std::convert::Infallible; +use std::fmt; +use std::{collections::HashMap, num::NonZeroU32}; #[cfg(feature = "unstable")] -use std::{fs::File, path::Path}; +use std::fs::File; use prost::Message; use rand_core::{OsRng, RngCore}; -use rusqlite::params; +use rusqlite::{params, Connection}; +use secrecy::SecretVec; +use tempfile::NamedTempFile; +#[cfg(feature = "unstable")] +use tempfile::TempDir; + +#[allow(deprecated)] +use zcash_client_backend::data_api::wallet::create_spend_to_address; use zcash_client_backend::{ - keys::{sapling, UnifiedFullViewingKey}, + address::RecipientAddress, + data_api::{ + self, + chain::{scan_cached_blocks, BlockSource}, + wallet::{ + create_proposed_transaction, + input_selection::{GreedyInputSelectorError, InputSelector, Proposal}, + propose_transfer, spend, + }, + }, + keys::{sapling, UnifiedFullViewingKey, UnifiedSpendingKey}, proto::compact_formats::{ self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }, + wallet::OvkPolicy, + zip321, }; use zcash_note_encryption::Domain; use zcash_primitives::{ @@ -25,19 +46,452 @@ use zcash_primitives::{ value::NoteValue, Note, Nullifier, PaymentAddress, }, - transaction::components::Amount, + transaction::{ + components::{amount::BalanceError, Amount}, + fees::FeeRule, + TxId, + }, zip32::{sapling::DiversifiableFullViewingKey, DiversifierIndex}, }; #[cfg(feature = "transparent-inputs")] -use zcash_primitives::{legacy, legacy::keys::IncomingViewingKey}; +use zcash_client_backend::data_api::wallet::shield_transparent_funds; +#[cfg(feature = "transparent-inputs")] +use zcash_primitives::{ + legacy, legacy::keys::IncomingViewingKey, transaction::components::amount::NonNegativeAmount, +}; -use crate::{wallet::init::init_accounts_table, AccountId, WalletDb}; +use crate::{ + chain::init::init_cache_database, + error::SqliteClientError, + wallet::{ + commitment_tree, + init::{init_accounts_table, init_wallet_db}, + sapling::tests::test_prover, + }, + AccountId, ReceivedNoteId, WalletDb, +}; use super::BlockDb; #[cfg(feature = "unstable")] -use super::chain::BlockMeta; +use crate::{ + chain::{init::init_blockmeta_db, BlockMeta}, + FsBlockDb, +}; +/// A builder for a `zcash_client_sqlite` test. +pub(crate) struct TestBuilder { + cache: Cache, + seed: Option>, + with_test_account: bool, +} + +impl TestBuilder<()> { + /// Constructs a new test. + pub(crate) fn new() -> Self { + TestBuilder { + cache: (), + seed: None, + with_test_account: false, + } + } + + /// Adds a [`BlockDb`] cache to the test. + pub(crate) fn with_block_cache(self) -> TestBuilder { + TestBuilder { + cache: BlockCache::new(), + seed: self.seed, + with_test_account: self.with_test_account, + } + } + + /// Adds a [`FsBlockDb`] cache to the test. + #[cfg(feature = "unstable")] + pub(crate) fn with_fs_block_cache(self) -> TestBuilder { + TestBuilder { + cache: FsBlockCache::new(), + seed: self.seed, + with_test_account: self.with_test_account, + } + } +} + +impl TestBuilder { + /// Gives the test knowledge of the wallet seed for initialization. + pub(crate) fn with_seed(mut self, seed: SecretVec) -> Self { + // TODO remove + self.seed = Some(seed); + self + } + + pub(crate) fn with_test_account(mut self) -> Self { + self.with_test_account = true; + self + } + + /// Builds the runner for this test. + pub(crate) fn build(self) -> TestRunner { + let params = network(); + + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), params).unwrap(); + init_wallet_db(&mut db_data, self.seed).unwrap(); + + let test_account = if self.with_test_account { + // Add an account to the wallet + Some(init_test_accounts_table_ufvk(&mut db_data)) + } else { + None + }; + + TestRunner { + params, + cache: self.cache, + latest_cached_block: None, + _data_file: data_file, + db_data, + test_account, + } + } +} + +/// A `zcash_client_sqlite` test runner. +pub(crate) struct TestRunner { + params: Network, + cache: Cache, + latest_cached_block: Option<(BlockHeight, BlockHash, u32)>, + _data_file: NamedTempFile, + db_data: WalletDb, + test_account: Option<(UnifiedFullViewingKey, Option)>, +} + +impl TestRunner +where + ::Error: fmt::Debug, +{ + /// Exposes an immutable reference to the test's [`BlockSource`]. + #[cfg(feature = "unstable")] + pub(crate) fn cache(&self) -> &Cache::BlockSource { + self.cache.block_source() + } + + /// Creates a fake block at the expected next height containing a single output of the + /// given value, and inserts it into the cache. + pub(crate) fn generate_next_block( + &mut self, + dfvk: &DiversifiableFullViewingKey, + req: AddressType, + value: Amount, + ) -> (BlockHeight, Cache::InsertResult, Nullifier) { + let (height, prev_hash, initial_sapling_tree_size) = self + .latest_cached_block + .map(|(prev_height, prev_hash, end_size)| (prev_height + 1, prev_hash, end_size)) + .unwrap_or_else(|| (sapling_activation_height(), BlockHash([0; 32]), 0)); + + let (res, nf) = self.generate_block_at( + height, + prev_hash, + dfvk, + req, + value, + initial_sapling_tree_size, + ); + + (height, res, nf) + } + + /// Creates a fake block with the given height and hash containing a single output of + /// the given value, and inserts it into the cache. + /// + /// This generated block will be treated as the latest block, and subsequent calls to + /// [`Self::generate_next_block`] will build on it. + pub(crate) fn generate_block_at( + &mut self, + height: BlockHeight, + prev_hash: BlockHash, + dfvk: &DiversifiableFullViewingKey, + req: AddressType, + value: Amount, + initial_sapling_tree_size: u32, + ) -> (Cache::InsertResult, Nullifier) { + let (cb, nf) = fake_compact_block( + height, + prev_hash, + dfvk, + req, + value, + initial_sapling_tree_size, + ); + let res = self.cache.insert(&cb); + + self.latest_cached_block = Some(( + height, + cb.hash(), + initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), + )); + + (res, nf) + } + + /// Creates a fake block at the expected next height spending the given note, and + /// inserts it into the cache. + pub(crate) fn generate_next_block_spending( + &mut self, + dfvk: &DiversifiableFullViewingKey, + note: (Nullifier, Amount), + to: PaymentAddress, + value: Amount, + ) -> (BlockHeight, Cache::InsertResult) { + let (height, prev_hash, initial_sapling_tree_size) = self + .latest_cached_block + .map(|(prev_height, prev_hash, end_size)| (prev_height + 1, prev_hash, end_size)) + .unwrap_or_else(|| (sapling_activation_height(), BlockHash([0; 32]), 0)); + + let cb = fake_compact_block_spending( + height, + prev_hash, + note, + dfvk, + to, + value, + initial_sapling_tree_size, + ); + let res = self.cache.insert(&cb); + + self.latest_cached_block = Some(( + height, + cb.hash(), + initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), + )); + + (height, res) + } + + /// Invokes [`scan_cached_blocks`] with the given arguments, expecting success. + pub(crate) fn scan_cached_blocks(&mut self, from_height: BlockHeight, limit: usize) { + self.try_scan_cached_blocks(from_height, limit) + .expect("should succeed for this test"); + } + + /// Invokes [`scan_cached_blocks`] with the given arguments. + pub(crate) fn try_scan_cached_blocks( + &mut self, + from_height: BlockHeight, + limit: usize, + ) -> Result< + (), + data_api::chain::error::Error< + SqliteClientError, + ::Error, + >, + > { + scan_cached_blocks( + &self.params, + self.cache.block_source(), + &mut self.db_data, + from_height, + limit, + ) + } +} + +impl TestRunner { + /// Exposes an immutable reference to the test's [`WalletDb`]. + pub(crate) fn wallet(&self) -> &WalletDb { + &self.db_data + } + + /// Exposes a mutable reference to the test's [`WalletDb`]. + pub(crate) fn wallet_mut(&mut self) -> &mut WalletDb { + &mut self.db_data + } + + /// Exposes the test account, if enabled via [`TestBuilder::with_test_account`]. + #[cfg(feature = "unstable")] + pub(crate) fn test_account( + &self, + ) -> Option<(UnifiedFullViewingKey, Option)> { + self.test_account.as_ref().cloned() + } + + /// Exposes the test account's Sapling DFVK, if enabled via [`TestBuilder::with_test_account`]. + pub(crate) fn test_account_sapling(&self) -> Option { + self.test_account + .as_ref() + .map(|(ufvk, _)| ufvk.sapling().unwrap().clone()) + } + + /// Invokes [`create_spend_to_address`] with the given arguments. + #[allow(deprecated)] + #[allow(clippy::type_complexity)] + pub(crate) fn create_spend_to_address( + &mut self, + usk: &UnifiedSpendingKey, + to: &RecipientAddress, + amount: Amount, + memo: Option, + ovk_policy: OvkPolicy, + min_confirmations: NonZeroU32, + ) -> Result< + TxId, + data_api::error::Error< + SqliteClientError, + commitment_tree::Error, + GreedyInputSelectorError, + Infallible, + ReceivedNoteId, + >, + > { + create_spend_to_address( + &mut self.db_data, + &self.params, + test_prover(), + usk, + to, + amount, + memo, + ovk_policy, + min_confirmations, + ) + } + + /// Invokes [`spend`] with the given arguments. + #[allow(clippy::type_complexity)] + pub(crate) fn spend( + &mut self, + input_selector: &InputsT, + usk: &UnifiedSpendingKey, + request: zip321::TransactionRequest, + ovk_policy: OvkPolicy, + min_confirmations: NonZeroU32, + ) -> Result< + TxId, + data_api::error::Error< + SqliteClientError, + commitment_tree::Error, + InputsT::Error, + ::Error, + ReceivedNoteId, + >, + > + where + InputsT: InputSelector>, + { + spend( + &mut self.db_data, + &self.params, + test_prover(), + input_selector, + usk, + request, + ovk_policy, + min_confirmations, + ) + } + + /// Invokes [`propose_transfer`] with the given arguments. + #[allow(clippy::type_complexity)] + pub(crate) fn propose_transfer( + &mut self, + spend_from_account: AccountId, + input_selector: &InputsT, + request: zip321::TransactionRequest, + min_confirmations: NonZeroU32, + ) -> Result< + Proposal, + data_api::error::Error< + SqliteClientError, + Infallible, + InputsT::Error, + ::Error, + ReceivedNoteId, + >, + > + where + InputsT: InputSelector>, + { + propose_transfer::<_, _, _, Infallible>( + &mut self.db_data, + &self.params, + spend_from_account, + input_selector, + request, + min_confirmations, + ) + } + + /// Invokes [`create_proposed_transaction`] with the given arguments. + pub(crate) fn create_proposed_transaction( + &mut self, + usk: &UnifiedSpendingKey, + ovk_policy: OvkPolicy, + proposal: Proposal, + min_confirmations: NonZeroU32, + change_memo: Option, + ) -> Result< + TxId, + data_api::error::Error< + SqliteClientError, + commitment_tree::Error, + Infallible, + FeeRuleT::Error, + ReceivedNoteId, + >, + > + where + FeeRuleT: FeeRule, + { + create_proposed_transaction::<_, _, Infallible, _>( + &mut self.db_data, + &self.params, + test_prover(), + usk, + ovk_policy, + proposal, + min_confirmations, + change_memo, + ) + } + + /// Invokes [`shield_transparent_funds`] with the given arguments. + #[cfg(feature = "transparent-inputs")] + #[allow(clippy::type_complexity)] + pub(crate) fn shield_transparent_funds( + &mut self, + input_selector: &InputsT, + shielding_threshold: NonNegativeAmount, + usk: &UnifiedSpendingKey, + from_addrs: &[TransparentAddress], + memo: &MemoBytes, + min_confirmations: NonZeroU32, + ) -> Result< + TxId, + data_api::error::Error< + SqliteClientError, + commitment_tree::Error, + InputsT::Error, + ::Error, + ReceivedNoteId, + >, + > + where + InputsT: InputSelector>, + { + shield_transparent_funds( + &mut self.db_data, + &self.params, + test_prover(), + input_selector, + shielding_threshold, + usk, + from_addrs, + memo, + min_confirmations, + ) + } +} #[cfg(feature = "mainnet")] pub(crate) fn network() -> Network { @@ -266,39 +720,103 @@ pub(crate) fn fake_compact_block_spending( cb } -/// Insert a fake CompactBlock into the cache DB. -pub(crate) fn insert_into_cache(db_cache: &BlockDb, cb: &CompactBlock) { - let cb_bytes = cb.encode_to_vec(); - db_cache - .0 - .prepare("INSERT INTO compactblocks (height, data) VALUES (?, ?)") - .unwrap() - .execute(params![u32::from(cb.height()), cb_bytes,]) - .unwrap(); +/// Trait used by tests that require a block cache. +pub(crate) trait TestCache { + type BlockSource: BlockSource; + type InsertResult; + + /// Exposes the block cache as a [`BlockSource`]. + fn block_source(&self) -> &Self::BlockSource; + + /// Inserts a CompactBlock into the cache DB. + fn insert(&self, cb: &CompactBlock) -> Self::InsertResult; +} + +pub(crate) struct BlockCache { + _cache_file: NamedTempFile, + db_cache: BlockDb, +} + +impl BlockCache { + fn new() -> Self { + let cache_file = NamedTempFile::new().unwrap(); + let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); + init_cache_database(&db_cache).unwrap(); + + BlockCache { + _cache_file: cache_file, + db_cache, + } + } +} + +impl TestCache for BlockCache { + type BlockSource = BlockDb; + type InsertResult = (); + + fn block_source(&self) -> &Self::BlockSource { + &self.db_cache + } + + fn insert(&self, cb: &CompactBlock) { + let cb_bytes = cb.encode_to_vec(); + self.db_cache + .0 + .prepare("INSERT INTO compactblocks (height, data) VALUES (?, ?)") + .unwrap() + .execute(params![u32::from(cb.height()), cb_bytes,]) + .unwrap(); + } } #[cfg(feature = "unstable")] -pub(crate) fn store_in_fsblockdb>( - fsblockdb_root: P, - cb: &CompactBlock, -) -> BlockMeta { - use std::io::Write; - - let meta = BlockMeta { - height: cb.height(), - block_hash: cb.hash(), - block_time: cb.time, - sapling_outputs_count: cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum(), - orchard_actions_count: cb.vtx.iter().map(|tx| tx.actions.len() as u32).sum(), - }; +pub(crate) struct FsBlockCache { + fsblockdb_root: TempDir, + db_meta: FsBlockDb, +} - let blocks_dir = fsblockdb_root.as_ref().join("blocks"); - let block_path = meta.block_file_path(&blocks_dir); +#[cfg(feature = "unstable")] +impl FsBlockCache { + fn new() -> Self { + let fsblockdb_root = tempfile::tempdir().unwrap(); + let mut db_meta = FsBlockDb::for_path(&fsblockdb_root).unwrap(); + init_blockmeta_db(&mut db_meta).unwrap(); + + FsBlockCache { + fsblockdb_root, + db_meta, + } + } +} - File::create(block_path) - .unwrap() - .write_all(&cb.encode_to_vec()) - .unwrap(); +#[cfg(feature = "unstable")] +impl TestCache for FsBlockCache { + type BlockSource = FsBlockDb; + type InsertResult = BlockMeta; + + fn block_source(&self) -> &Self::BlockSource { + &self.db_meta + } + + fn insert(&self, cb: &CompactBlock) -> Self::InsertResult { + use std::io::Write; + + let meta = BlockMeta { + height: cb.height(), + block_hash: cb.hash(), + block_time: cb.time, + sapling_outputs_count: cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum(), + orchard_actions_count: cb.vtx.iter().map(|tx| tx.actions.len() as u32).sum(), + }; + + let blocks_dir = self.fsblockdb_root.as_ref().join("blocks"); + let block_path = meta.block_file_path(&blocks_dir); + + File::create(block_path) + .unwrap() + .write_all(&cb.encode_to_vec()) + .unwrap(); - meta + meta + } } From 84a5d5ca94dfb733b57d799d949b466cb27e7d6c Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 28 Aug 2023 16:31:02 +0000 Subject: [PATCH 0111/1122] zcash_client_sqlite: Migrate all tests to `TestBuilder` --- zcash_client_sqlite/src/chain.rs | 470 ++++------------- zcash_client_sqlite/src/lib.rs | 121 ++--- zcash_client_sqlite/src/testing.rs | 8 - zcash_client_sqlite/src/wallet.rs | 60 +-- zcash_client_sqlite/src/wallet/sapling.rs | 577 +++++---------------- zcash_client_sqlite/src/wallet/scanning.rs | 95 ++-- 6 files changed, 337 insertions(+), 994 deletions(-) diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index e23321eb73..35f90a80a7 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -325,7 +325,6 @@ mod tests { use std::num::NonZeroU32; use secrecy::Secret; - use tempfile::NamedTempFile; use zcash_primitives::{ block::BlockHash, @@ -336,9 +335,8 @@ mod tests { use zcash_client_backend::{ address::RecipientAddress, data_api::{ - chain::{error::Error, scan_cached_blocks}, - wallet::{input_selection::GreedyInputSelector, spend}, - WalletRead, WalletWrite, + chain::error::Error, wallet::input_selection::GreedyInputSelector, WalletRead, + WalletWrite, }, fees::{zip317::SingleOutputChangeStrategy, DustOutputPolicy}, scanning::ScanError, @@ -347,340 +345,192 @@ mod tests { }; use crate::{ - chain::init::init_cache_database, - testing::{ - self, fake_compact_block, fake_compact_block_spending, init_test_accounts_table, - insert_into_cache, sapling_activation_height, AddressType, - }, - wallet::{get_balance, init::init_wallet_db, truncate_to_height}, - AccountId, BlockDb, WalletDb, + testing::{AddressType, TestBuilder}, + wallet::{get_balance, truncate_to_height}, + AccountId, }; #[test] fn valid_chain_states() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); - init_cache_database(&db_cache).unwrap(); - - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); + let mut test = TestBuilder::new() + .with_block_cache() + .with_seed(Secret::new(vec![])) + .with_test_account() + .build(); - // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); + let dfvk = test.test_account_sapling().unwrap(); // Empty chain should return None - assert_matches!(db_data.chain_height(), Ok(None)); + assert_matches!(test.wallet().chain_height(), Ok(None)); // Create a fake CompactBlock sending value to the address - let (cb, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), + let (h1, _, _) = test.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), - 0, ); - insert_into_cache(&db_cache, &cb); - // Scan the cache - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 1, - ) - .unwrap(); + test.scan_cached_blocks(h1, 1); // Create a second fake CompactBlock sending more value to the address - let (cb2, _) = fake_compact_block( - sapling_activation_height() + 1, - cb.hash(), + let (h2, _, _) = test.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(7).unwrap(), - 1, ); - insert_into_cache(&db_cache, &cb2); - // Scanning should detect no inconsistencies - assert_matches!( - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height() + 1, - 1, - ), - Ok(()) - ); + test.scan_cached_blocks(h2, 1); } #[test] fn invalid_chain_cache_disconnected() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); - init_cache_database(&db_cache).unwrap(); + let mut test = TestBuilder::new() + .with_block_cache() + .with_seed(Secret::new(vec![])) + .with_test_account() + .build(); - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); - - // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); + let dfvk = test.test_account_sapling().unwrap(); // Create some fake CompactBlocks - let (cb, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), + let (h, _, _) = test.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), - 0, ); - let (cb2, _) = fake_compact_block( - sapling_activation_height() + 1, - cb.hash(), + let (last_contiguous_height, _, _) = test.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(7).unwrap(), - 1, ); - insert_into_cache(&db_cache, &cb); - insert_into_cache(&db_cache, &cb2); // Scanning the cache should find no inconsistencies - assert_matches!( - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 2, - ), - Ok(()) - ); + test.scan_cached_blocks(h, 2); // Create more fake CompactBlocks that don't connect to the scanned ones - let (cb3, _) = fake_compact_block( - sapling_activation_height() + 2, + let disconnect_height = last_contiguous_height + 1; + test.generate_block_at( + disconnect_height, BlockHash([1; 32]), &dfvk, AddressType::DefaultExternal, Amount::from_u64(8).unwrap(), 2, ); - let (cb4, _) = fake_compact_block( - sapling_activation_height() + 3, - cb3.hash(), + test.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(3).unwrap(), - 3, ); - insert_into_cache(&db_cache, &cb3); - insert_into_cache(&db_cache, &cb4); // Data+cache chain should be invalid at the data/cache boundary assert_matches!( - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height() + 2, + test.try_scan_cached_blocks( + disconnect_height, 2 ), Err(Error::Scan(ScanError::PrevHashMismatch { at_height })) - if at_height == sapling_activation_height() + 2 + if at_height == disconnect_height ); } #[test] fn data_db_truncation() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); - init_cache_database(&db_cache).unwrap(); + let mut test = TestBuilder::new() + .with_block_cache() + .with_seed(Secret::new(vec![])) + .with_test_account() + .build(); - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); - - // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); + let dfvk = test.test_account_sapling().unwrap(); // Account balance should be zero assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); // Create fake CompactBlocks sending value to the address let value = Amount::from_u64(5).unwrap(); let value2 = Amount::from_u64(7).unwrap(); - let (cb, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), - &dfvk, - AddressType::DefaultExternal, - value, - 0, - ); - - let (cb2, _) = fake_compact_block( - sapling_activation_height() + 1, - cb.hash(), - &dfvk, - AddressType::DefaultExternal, - value2, - 1, - ); - insert_into_cache(&db_cache, &cb); - insert_into_cache(&db_cache, &cb2); + let (h, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + test.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); // Scan the cache - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 2, - ) - .unwrap(); + test.scan_cached_blocks(h, 2); // Account balance should reflect both received notes assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), (value + value2).unwrap() ); // "Rewind" to height of last scanned block - db_data - .transactionally(|wdb| { - truncate_to_height(wdb.conn.0, &wdb.params, sapling_activation_height() + 1) - }) + test.wallet_mut() + .transactionally(|wdb| truncate_to_height(wdb.conn.0, &wdb.params, h + 1)) .unwrap(); // Account balance should be unaltered assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), (value + value2).unwrap() ); // Rewind so that one block is dropped - db_data - .transactionally(|wdb| { - truncate_to_height(wdb.conn.0, &wdb.params, sapling_activation_height()) - }) + test.wallet_mut() + .transactionally(|wdb| truncate_to_height(wdb.conn.0, &wdb.params, h)) .unwrap(); // Account balance should only contain the first received note assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), value ); // Scan the cache again - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 2, - ) - .unwrap(); + test.scan_cached_blocks(h, 2); // Account balance should again reflect both received notes assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), (value + value2).unwrap() ); } #[test] fn scan_cached_blocks_allows_blocks_out_of_order() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); - init_cache_database(&db_cache).unwrap(); - - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); + let mut test = TestBuilder::new() + .with_block_cache() + .with_seed(Secret::new(vec![])) + .build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = db_data.create_account(&seed).unwrap(); + let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Create a block with height SAPLING_ACTIVATION_HEIGHT let value = Amount::from_u64(50000).unwrap(); - let (cb1, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), - &dfvk, - AddressType::DefaultExternal, - value, - 0, - ); - insert_into_cache(&db_cache, &cb1); - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 1, - ) - .unwrap(); + let (h1, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + test.scan_cached_blocks(h1, 1); assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), value ); // Create blocks to reach SAPLING_ACTIVATION_HEIGHT + 2 - let (cb2, _) = fake_compact_block( - sapling_activation_height() + 1, - cb1.hash(), - &dfvk, - AddressType::DefaultExternal, - value, - 1, - ); - let (cb3, _) = fake_compact_block( - sapling_activation_height() + 2, - cb2.hash(), - &dfvk, - AddressType::DefaultExternal, - value, - 2, - ); + let (h2, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let (h3, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); // Scan the later block first - insert_into_cache(&db_cache, &cb3); - assert_matches!( - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height() + 2, - 1 - ), - Ok(_) - ); + test.scan_cached_blocks(h3, 1); - // If we add a block of height SAPLING_ACTIVATION_HEIGHT + 1, we can now scan that - insert_into_cache(&db_cache, &cb2); - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height() + 1, - 1, - ) - .unwrap(); + // Now scan the block of height SAPLING_ACTIVATION_HEIGHT + 1 + test.scan_cached_blocks(h2, 1); assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), Amount::from_u64(150_000).unwrap() ); @@ -699,10 +549,7 @@ mod tests { DustOutputPolicy::default(), ); assert_matches!( - spend( - &mut db_data, - &testing::network(), - crate::wallet::sapling::tests::test_prover(), + test.spend( &input_selector, &usk, req, @@ -715,124 +562,74 @@ mod tests { #[test] fn scan_cached_blocks_finds_received_notes() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); - init_cache_database(&db_cache).unwrap(); + let mut test = TestBuilder::new() + .with_block_cache() + .with_seed(Secret::new(vec![])) + .with_test_account() + .build(); - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); - - // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); + let dfvk = test.test_account_sapling().unwrap(); // Account balance should be zero assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); // Create a fake CompactBlock sending value to the address let value = Amount::from_u64(5).unwrap(); - let (cb, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), - &dfvk, - AddressType::DefaultExternal, - value, - 0, - ); - insert_into_cache(&db_cache, &cb); + let (h1, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); // Scan the cache - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 1, - ) - .unwrap(); + test.scan_cached_blocks(h1, 1); // Account balance should reflect the received note assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), value ); // Create a second fake CompactBlock sending more value to the address let value2 = Amount::from_u64(7).unwrap(); - let (cb2, _) = fake_compact_block( - sapling_activation_height() + 1, - cb.hash(), - &dfvk, - AddressType::DefaultExternal, - value2, - 1, - ); - insert_into_cache(&db_cache, &cb2); + let (h2, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); // Scan the cache again - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height() + 1, - 1, - ) - .unwrap(); + test.scan_cached_blocks(h2, 1); // Account balance should reflect both received notes assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), (value + value2).unwrap() ); } #[test] fn scan_cached_blocks_finds_change_notes() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); - init_cache_database(&db_cache).unwrap(); + let mut test = TestBuilder::new() + .with_block_cache() + .with_seed(Secret::new(vec![])) + .with_test_account() + .build(); - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); - - // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); + let dfvk = test.test_account_sapling().unwrap(); // Account balance should be zero assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); // Create a fake CompactBlock sending value to the address let value = Amount::from_u64(5).unwrap(); - let (cb, nf) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), - &dfvk, - AddressType::DefaultExternal, - value, - 0, - ); - insert_into_cache(&db_cache, &cb); + let (received_height, _, nf) = + test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); // Scan the cache - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 1, - ) - .unwrap(); + test.scan_cached_blocks(received_height, 1); // Account balance should reflect the received note assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), value ); @@ -840,113 +637,60 @@ mod tests { let extsk2 = ExtendedSpendingKey::master(&[0]); let to2 = extsk2.default_address().1; let value2 = Amount::from_u64(2).unwrap(); - insert_into_cache( - &db_cache, - &fake_compact_block_spending( - sapling_activation_height() + 1, - cb.hash(), - (nf, value), - &dfvk, - to2, - value2, - 1, - ), - ); + let (spent_height, _) = test.generate_next_block_spending(&dfvk, (nf, value), to2, value2); // Scan the cache again - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height() + 1, - 1, - ) - .unwrap(); + test.scan_cached_blocks(spent_height, 1); // Account balance should equal the change assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), (value - value2).unwrap() ); } #[test] fn scan_cached_blocks_detects_spends_out_of_order() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); - init_cache_database(&db_cache).unwrap(); + let mut test = TestBuilder::new() + .with_block_cache() + .with_seed(Secret::new(vec![])) + .with_test_account() + .build(); - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); - - // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); + let dfvk = test.test_account_sapling().unwrap(); // Account balance should be zero assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); // Create a fake CompactBlock sending value to the address let value = Amount::from_u64(5).unwrap(); - let (cb, nf) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), - &dfvk, - AddressType::DefaultExternal, - value, - 0, - ); - insert_into_cache(&db_cache, &cb); + let (received_height, _, nf) = + test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); // Create a second fake CompactBlock spending value from the address let extsk2 = ExtendedSpendingKey::master(&[0]); let to2 = extsk2.default_address().1; let value2 = Amount::from_u64(2).unwrap(); - insert_into_cache( - &db_cache, - &fake_compact_block_spending( - sapling_activation_height() + 1, - cb.hash(), - (nf, value), - &dfvk, - to2, - value2, - 1, - ), - ); + let (spent_height, _) = test.generate_next_block_spending(&dfvk, (nf, value), to2, value2); // Scan the spending block first. - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height() + 1, - 1, - ) - .unwrap(); + test.scan_cached_blocks(spent_height, 1); // Account balance should equal the change assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), (value - value2).unwrap() ); // Now scan the block in which we received the note that was spent. - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 1, - ) - .unwrap(); + test.scan_cached_blocks(received_height, 1); // Account balance should be the same. assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), (value - value2).unwrap() ); } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index df4f40785e..e3ecf3ecac 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -1087,51 +1087,33 @@ extern crate assert_matches; mod tests { use zcash_client_backend::data_api::{WalletRead, WalletWrite}; - use crate::{ - testing::{init_test_accounts_table_ufvk, network}, - wallet::init::init_wallet_db, - AccountId, WalletDb, - }; + use crate::{testing::TestBuilder, AccountId}; #[cfg(feature = "unstable")] - use zcash_primitives::{ - block::BlockHash, - consensus::{BlockHeight, Parameters}, - transaction::components::Amount, - }; + use zcash_primitives::{consensus::Parameters, transaction::components::Amount}; #[cfg(feature = "unstable")] use zcash_client_backend::keys::sapling; #[cfg(feature = "unstable")] - use crate::{ - chain::init::init_blockmeta_db, - testing::{fake_compact_block, store_in_fsblockdb, AddressType}, - FsBlockDb, - }; - - #[cfg(feature = "unstable")] - use super::BlockDb; + use crate::testing::AddressType; #[test] pub(crate) fn get_next_available_address() { - use tempfile::NamedTempFile; - - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), network()).unwrap(); + let mut test = TestBuilder::new().with_test_account().build(); let account = AccountId::from(0); - init_wallet_db(&mut db_data, None).unwrap(); - init_test_accounts_table_ufvk(&mut db_data); - - let current_addr = db_data.get_current_address(account).unwrap(); + let current_addr = test.wallet().get_current_address(account).unwrap(); assert!(current_addr.is_some()); - let addr2 = db_data.get_next_available_address(account).unwrap(); + let addr2 = test + .wallet_mut() + .get_next_available_address(account) + .unwrap(); assert!(addr2.is_some()); assert_ne!(current_addr, addr2); - let addr2_cur = db_data.get_current_address(account).unwrap(); + let addr2_cur = test.wallet().get_current_address(account).unwrap(); assert_eq!(addr2, addr2_cur); } @@ -1139,25 +1121,18 @@ mod tests { #[test] fn transparent_receivers() { use secrecy::Secret; - use tempfile::NamedTempFile; - - use crate::{ - chain::init::init_cache_database, testing::network, wallet::init::init_wallet_db, - }; - - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); - init_cache_database(&db_cache).unwrap(); - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); + let test = TestBuilder::new() + .with_block_cache() + .with_seed(Secret::new(vec![])) + .with_test_account() + .build(); // Add an account to the wallet. - let (ufvk, taddr) = init_test_accounts_table_ufvk(&mut db_data); + let (ufvk, taddr) = test.test_account().unwrap(); let taddr = taddr.unwrap(); - let receivers = db_data.get_transparent_receivers(0.into()).unwrap(); + let receivers = test.wallet().get_transparent_receivers(0.into()).unwrap(); // The receiver for the default UA should be in the set. assert!(receivers.contains_key(ufvk.default_address().0.transparent().unwrap())); @@ -1169,74 +1144,44 @@ mod tests { #[cfg(feature = "unstable")] #[test] pub(crate) fn fsblockdb_api() { - // Initialise a BlockMeta DB in a new directory. - let fsblockdb_root = tempfile::tempdir().unwrap(); - let mut db_meta = FsBlockDb::for_path(&fsblockdb_root).unwrap(); - init_blockmeta_db(&mut db_meta).unwrap(); + let mut test = TestBuilder::new().with_fs_block_cache().build(); // The BlockMeta DB starts off empty. - assert_eq!(db_meta.get_max_cached_height().unwrap(), None); + assert_eq!(test.cache().get_max_cached_height().unwrap(), None); // Generate some fake CompactBlocks. let seed = [0u8; 32]; let account = AccountId::from(0); - let extsk = sapling::spending_key(&seed, network().coin_type(), account); + let extsk = sapling::spending_key(&seed, test.wallet().params.coin_type(), account); let dfvk = extsk.to_diversifiable_full_viewing_key(); - let (cb1, _) = fake_compact_block( - BlockHeight::from_u32(1), - BlockHash([1; 32]), + let (h1, meta1, _) = test.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), - 0, ); - let (cb2, _) = fake_compact_block( - BlockHeight::from_u32(2), - BlockHash([2; 32]), + let (h2, meta2, _) = test.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(10).unwrap(), - 1, ); - // Write the CompactBlocks to the BlockMeta DB's corresponding disk storage. - let meta1 = store_in_fsblockdb(&fsblockdb_root, &cb1); - let meta2 = store_in_fsblockdb(&fsblockdb_root, &cb2); - // The BlockMeta DB is not updated until we do so explicitly. - assert_eq!(db_meta.get_max_cached_height().unwrap(), None); + assert_eq!(test.cache().get_max_cached_height().unwrap(), None); // Inform the BlockMeta DB about the newly-persisted CompactBlocks. - db_meta.write_block_metadata(&[meta1, meta2]).unwrap(); + test.cache().write_block_metadata(&[meta1, meta2]).unwrap(); // The BlockMeta DB now sees blocks up to height 2. - assert_eq!( - db_meta.get_max_cached_height().unwrap(), - Some(BlockHeight::from_u32(2)), - ); - assert_eq!( - db_meta.find_block(BlockHeight::from_u32(1)).unwrap(), - Some(meta1), - ); - assert_eq!( - db_meta.find_block(BlockHeight::from_u32(2)).unwrap(), - Some(meta2), - ); - assert_eq!(db_meta.find_block(BlockHeight::from_u32(3)).unwrap(), None); + assert_eq!(test.cache().get_max_cached_height().unwrap(), Some(h2),); + assert_eq!(test.cache().find_block(h1).unwrap(), Some(meta1)); + assert_eq!(test.cache().find_block(h2).unwrap(), Some(meta2)); + assert_eq!(test.cache().find_block(h2 + 1).unwrap(), None); // Rewinding to height 1 should cause the metadata for height 2 to be deleted. - db_meta - .truncate_to_height(BlockHeight::from_u32(1)) - .unwrap(); - assert_eq!( - db_meta.get_max_cached_height().unwrap(), - Some(BlockHeight::from_u32(1)), - ); - assert_eq!( - db_meta.find_block(BlockHeight::from_u32(1)).unwrap(), - Some(meta1), - ); - assert_eq!(db_meta.find_block(BlockHeight::from_u32(2)).unwrap(), None); - assert_eq!(db_meta.find_block(BlockHeight::from_u32(3)).unwrap(), None); + test.cache().truncate_to_height(h1).unwrap(); + assert_eq!(test.cache().get_max_cached_height().unwrap(), Some(h1)); + assert_eq!(test.cache().find_block(h1).unwrap(), Some(meta1)); + assert_eq!(test.cache().find_block(h2).unwrap(), None); + assert_eq!(test.cache().find_block(h2 + 1).unwrap(), None); } } diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 5c0cbee8fe..db6a135fbc 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -517,14 +517,6 @@ pub(crate) fn sapling_activation_height() -> BlockHeight { .unwrap() } -#[cfg(test)] -pub(crate) fn init_test_accounts_table( - db_data: &mut WalletDb, -) -> (DiversifiableFullViewingKey, Option) { - let (ufvk, taddr) = init_test_accounts_table_ufvk(db_data); - (ufvk.sapling().unwrap().clone(), taddr) -} - #[cfg(test)] pub(crate) fn init_test_accounts_table_ufvk( db_data: &mut WalletDb, diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 8fb7ebe708..d2fdbc639f 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1572,17 +1572,12 @@ mod tests { use std::num::NonZeroU32; use secrecy::Secret; - use tempfile::NamedTempFile; use zcash_primitives::transaction::components::Amount; use zcash_client_backend::data_api::WalletRead; - use crate::{ - testing, - wallet::{get_current_address, init::init_wallet_db}, - AccountId, WalletDb, - }; + use crate::{testing::TestBuilder, AccountId}; use super::get_balance; @@ -1599,22 +1594,20 @@ mod tests { #[test] fn empty_database_has_no_balance() { - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); - - // Add an account to the wallet - testing::init_test_accounts_table(&mut db_data); + let test = TestBuilder::new() + .with_seed(Secret::new(vec![])) + .with_test_account() + .build(); // The account should be empty assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); // We can't get an anchor height, as we have not scanned any blocks. assert_eq!( - db_data + test.wallet() .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) .unwrap(), None @@ -1622,11 +1615,11 @@ mod tests { // An invalid account has zero balance assert_matches!( - get_current_address(&db_data.conn, &db_data.params, AccountId::from(1)), + test.wallet().get_current_address(AccountId::from(1)), Ok(None) ); assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); } @@ -1634,17 +1627,20 @@ mod tests { #[test] #[cfg(feature = "transparent-inputs")] fn put_received_transparent_utxo() { - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, None).unwrap(); + let mut test = TestBuilder::new().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (account_id, _usk) = db_data.create_account(&seed).unwrap(); - let uaddr = db_data.get_current_address(account_id).unwrap().unwrap(); + let (account_id, _usk) = test.wallet_mut().create_account(&seed).unwrap(); + let uaddr = test + .wallet() + .get_current_address(account_id) + .unwrap() + .unwrap(); let taddr = uaddr.transparent().unwrap(); - let bal_absent = db_data + let bal_absent = test + .wallet() .get_transparent_balances(account_id, BlockHeight::from_u32(12345)) .unwrap(); assert!(bal_absent.is_empty()); @@ -1659,7 +1655,7 @@ mod tests { ) .unwrap(); - let res0 = super::put_received_transparent_utxo(&db_data.conn, &db_data.params, &utxo); + let res0 = test.wallet_mut().put_received_transparent_utxo(&utxo); assert_matches!(res0, Ok(_)); // Change the mined height of the UTXO and upsert; we should get back @@ -1673,13 +1669,11 @@ mod tests { BlockHeight::from_u32(34567), ) .unwrap(); - let res1 = super::put_received_transparent_utxo(&db_data.conn, &db_data.params, &utxo2); + let res1 = test.wallet_mut().put_received_transparent_utxo(&utxo2); assert_matches!(res1, Ok(id) if id == res0.unwrap()); assert_matches!( - super::get_unspent_transparent_outputs( - &db_data.conn, - &db_data.params, + test.wallet().get_unspent_transparent_outputs( taddr, BlockHeight::from_u32(12345), &[] @@ -1688,9 +1682,7 @@ mod tests { ); assert_matches!( - super::get_unspent_transparent_outputs( - &db_data.conn, - &db_data.params, + test.wallet().get_unspent_transparent_outputs( taddr, BlockHeight::from_u32(34567), &[] @@ -1702,21 +1694,21 @@ mod tests { ); assert_matches!( - db_data.get_transparent_balances(account_id, BlockHeight::from_u32(34567)), + test.wallet().get_transparent_balances(account_id, BlockHeight::from_u32(34567)), Ok(h) if h.get(taddr) == Amount::from_u64(100000).ok().as_ref() ); // Artificially delete the address from the addresses table so that // we can ensure the update fails if the join doesn't work. - db_data + test.wallet() .conn .execute( "DELETE FROM addresses WHERE cached_transparent_receiver_address = ?", - [Some(taddr.encode(&db_data.params))], + [Some(taddr.encode(&test.wallet().params))], ) .unwrap(); - let res2 = super::put_received_transparent_utxo(&db_data.conn, &db_data.params, &utxo2); + let res2 = test.wallet_mut().put_received_transparent_utxo(&utxo2); assert_matches!(res2, Err(_)); } } diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 0bfc60905b..8d95fcaea4 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -403,14 +403,11 @@ pub(crate) fn put_received_note( pub(crate) mod tests { use std::{convert::Infallible, num::NonZeroU32}; - use rusqlite::Connection; use secrecy::Secret; - use tempfile::NamedTempFile; use zcash_proofs::prover::LocalTxProver; use zcash_primitives::{ - block::BlockHash, consensus::BranchId, legacy::TransparentAddress, memo::{Memo, MemoBytes}, @@ -429,13 +426,8 @@ pub(crate) mod tests { address::RecipientAddress, data_api::{ self, - chain::scan_cached_blocks, error::Error, - wallet::{ - create_proposed_transaction, create_spend_to_address, - input_selection::{GreedyInputSelector, GreedyInputSelectorError}, - propose_transfer, spend, - }, + wallet::input_selection::{GreedyInputSelector, GreedyInputSelectorError}, ShieldedProtocol, WalletRead, WalletWrite, }, decrypt_transaction, @@ -446,21 +438,15 @@ pub(crate) mod tests { }; use crate::{ - chain::init::init_cache_database, error::SqliteClientError, - testing::{ - self, fake_compact_block, insert_into_cache, network, sapling_activation_height, - AddressType, - }, - wallet::{commitment_tree, get_balance, get_balance_at, init::init_wallet_db}, - AccountId, BlockDb, NoteId, ReceivedNoteId, WalletDb, + testing::{self, network, AddressType, BlockCache, TestBuilder, TestRunner}, + wallet::{commitment_tree, get_balance, get_balance_at}, + AccountId, NoteId, ReceivedNoteId, }; #[cfg(feature = "transparent-inputs")] use { - zcash_client_backend::{ - data_api::wallet::shield_transparent_funds, wallet::WalletTransparentOutput, - }, + zcash_client_backend::wallet::WalletTransparentOutput, zcash_primitives::transaction::components::{amount::NonNegativeAmount, OutPoint, TxOut}, }; @@ -475,50 +461,30 @@ pub(crate) mod tests { #[test] fn send_proposed_transfer() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb(Connection::open(cache_file.path()).unwrap()); - init_cache_database(&db_cache).unwrap(); - - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, None).unwrap(); + let mut test = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (account, usk) = db_data.create_account(&seed).unwrap(); + let (account, usk) = test.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note let value = Amount::from_u64(60000).unwrap(); - let (cb, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), - &dfvk, - AddressType::DefaultExternal, - value, - 0, - ); - insert_into_cache(&db_cache, &cb); - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 1, - ) - .unwrap(); + let (h, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + test.scan_cached_blocks(h, 1); // Verified balance matches total balance - let (_, anchor_height) = db_data + let (_, anchor_height) = test + .wallet() .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) .unwrap() .unwrap(); assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), value ); assert_eq!( - get_balance_at(&db_data.conn, AccountId::from(0), anchor_height).unwrap(), + get_balance_at(&test.wallet().conn, AccountId::from(0), anchor_height).unwrap(), value ); @@ -538,9 +504,7 @@ pub(crate) mod tests { let change_strategy = fixed::SingleOutputChangeStrategy::new(fee_rule); let input_selector = &GreedyInputSelector::new(change_strategy, DustOutputPolicy::default()); - let proposal_result = propose_transfer::<_, _, _, Infallible>( - &mut db_data, - &testing::network(), + let proposal_result = test.propose_transfer( account, input_selector, request, @@ -549,10 +513,7 @@ pub(crate) mod tests { assert_matches!(proposal_result, Ok(_)); let change_memo = "Test change memo".parse::().unwrap(); - let create_proposed_result = create_proposed_transaction::<_, _, Infallible, _>( - &mut db_data, - &testing::network(), - test_prover(), + let create_proposed_result = test.create_proposed_transaction( &usk, OvkPolicy::Sender, proposal_result.unwrap(), @@ -564,18 +525,14 @@ pub(crate) mod tests { let sent_tx_id = create_proposed_result.unwrap(); // Verify that the sent transaction was stored and that we can decrypt the memos - let tx = db_data + let tx = test + .wallet() .get_transaction(sent_tx_id) .expect("Created transaction was stored."); let ufvks = [(account, usk.to_unified_full_viewing_key())] .into_iter() .collect(); - let decrypted_outputs = decrypt_transaction( - &testing::network(), - sapling_activation_height() + 1, - &tx, - &ufvks, - ); + let decrypted_outputs = decrypt_transaction(&testing::network(), h + 1, &tx, &ufvks); assert_eq!(decrypted_outputs.len(), 2); let mut found_tx_change_memo = false; @@ -592,7 +549,8 @@ pub(crate) mod tests { assert!(found_tx_empty_memo); // Verify that the stored sent notes match what we're expecting - let mut stmt_sent_notes = db_data + let mut stmt_sent_notes = test + .wallet() .conn .prepare( "SELECT output_index @@ -622,7 +580,8 @@ pub(crate) mod tests { let mut found_sent_change_memo = false; let mut found_sent_empty_memo = false; for sent_note_id in sent_note_ids { - match db_data + match test + .wallet() .get_memo(sent_note_id) .expect("Note id is valid") .as_ref() @@ -642,20 +601,19 @@ pub(crate) mod tests { // Check that querying for a nonexistent sent note returns None assert_matches!( - db_data.get_memo(NoteId::new(sent_tx_id, ShieldedProtocol::Sapling, 12345)), + test.wallet() + .get_memo(NoteId::new(sent_tx_id, ShieldedProtocol::Sapling, 12345)), Ok(None) ); } #[test] fn create_to_address_fails_on_incorrect_usk() { - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); + let mut test = TestBuilder::new().with_seed(Secret::new(vec![])).build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = db_data.create_account(&seed).unwrap(); + let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); let to = dfvk.default_address().1.into(); @@ -665,10 +623,7 @@ pub(crate) mod tests { // Attempting to spend with a USK that is not in the wallet results in an error assert_matches!( - create_spend_to_address( - &mut db_data, - &testing::network(), - test_prover(), + test.create_spend_to_address( &usk1, &to, Amount::from_u64(1).unwrap(), @@ -682,28 +637,23 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_with_no_blocks() { - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, None).unwrap(); + let mut test = TestBuilder::new().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = db_data.create_account(&seed).unwrap(); + let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); let to = dfvk.default_address().1.into(); // Account balance should be zero assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); // We cannot do anything if we aren't synchronised assert_matches!( - create_spend_to_address( - &mut db_data, - &testing::network(), - test_prover(), + test.create_spend_to_address( &usk, &to, Amount::from_u64(1).unwrap(), @@ -717,84 +667,49 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_on_unverified_notes() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb(Connection::open(cache_file.path()).unwrap()); - init_cache_database(&db_cache).unwrap(); - - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, None).unwrap(); + let mut test = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = db_data.create_account(&seed).unwrap(); + let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); - let (mut cb, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), - &dfvk, - AddressType::DefaultExternal, - value, - 0, - ); - insert_into_cache(&db_cache, &cb); - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 1, - ) - .unwrap(); + let (h1, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + test.scan_cached_blocks(h1, 1); // Verified balance matches total balance - let (_, anchor_height) = db_data + let (_, anchor_height) = test + .wallet() .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) .unwrap() .unwrap(); assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), value ); assert_eq!( - get_balance_at(&db_data.conn, AccountId::from(0), anchor_height).unwrap(), + get_balance_at(&test.wallet().conn, AccountId::from(0), anchor_height).unwrap(), value ); // Add more funds to the wallet in a second note - cb = fake_compact_block( - sapling_activation_height() + 1, - cb.hash(), - &dfvk, - AddressType::DefaultExternal, - value, - 1, - ) - .0; - insert_into_cache(&db_cache, &cb); - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height() + 1, - 1, - ) - .unwrap(); + let (h2, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + test.scan_cached_blocks(h2, 1); // Verified balance does not include the second note - let (_, anchor_height2) = db_data + let (_, anchor_height2) = test + .wallet() .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) .unwrap() .unwrap(); assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), (value + value).unwrap() ); assert_eq!( - get_balance_at(&db_data.conn, AccountId::from(0), anchor_height2).unwrap(), + get_balance_at(&test.wallet().conn, AccountId::from(0), anchor_height2).unwrap(), value ); @@ -802,10 +717,7 @@ pub(crate) mod tests { let extsk2 = ExtendedSpendingKey::master(&[]); let to = extsk2.default_address().1.into(); assert_matches!( - create_spend_to_address( - &mut db_data, - &testing::network(), - test_prover(), + test.create_spend_to_address( &usk, &to, Amount::from_u64(70000).unwrap(), @@ -823,33 +735,14 @@ pub(crate) mod tests { // Mine blocks SAPLING_ACTIVATION_HEIGHT + 2 to 9 until just before the second // note is verified - for i in 2..10 { - cb = fake_compact_block( - sapling_activation_height() + i, - cb.hash(), - &dfvk, - AddressType::DefaultExternal, - value, - i, - ) - .0; - insert_into_cache(&db_cache, &cb); + for _ in 2..10 { + test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); } - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height() + 2, - 8, - ) - .unwrap(); + test.scan_cached_blocks(h2 + 1, 8); // Second spend still fails assert_matches!( - create_spend_to_address( - &mut db_data, - &testing::network(), - test_prover(), + test.create_spend_to_address( &usk, &to, Amount::from_u64(70000).unwrap(), @@ -866,31 +759,12 @@ pub(crate) mod tests { ); // Mine block 11 so that the second note becomes verified - cb = fake_compact_block( - sapling_activation_height() + 10, - cb.hash(), - &dfvk, - AddressType::DefaultExternal, - value, - 10, - ) - .0; - insert_into_cache(&db_cache, &cb); - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height() + 10, - 1, - ) - .unwrap(); + let (h11, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + test.scan_cached_blocks(h11, 1); // Second spend should now succeed assert_matches!( - create_spend_to_address( - &mut db_data, - &testing::network(), - test_prover(), + test.create_spend_to_address( &usk, &to, Amount::from_u64(70000).unwrap(), @@ -904,40 +778,22 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_on_locked_notes() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb(Connection::open(cache_file.path()).unwrap()); - init_cache_database(&db_cache).unwrap(); - - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); + let mut test = TestBuilder::new() + .with_block_cache() + .with_seed(Secret::new(vec![])) + .build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = db_data.create_account(&seed).unwrap(); + let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); - let (mut cb, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), - &dfvk, - AddressType::DefaultExternal, - value, - 0, - ); - insert_into_cache(&db_cache, &cb); - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 1, - ) - .unwrap(); + let (h1, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + test.scan_cached_blocks(h1, 1); assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), value ); @@ -945,10 +801,7 @@ pub(crate) mod tests { let extsk2 = ExtendedSpendingKey::master(&[]); let to = extsk2.default_address().1.into(); assert_matches!( - create_spend_to_address( - &mut db_data, - &testing::network(), - test_prover(), + test.create_spend_to_address( &usk, &to, Amount::from_u64(15000).unwrap(), @@ -961,10 +814,7 @@ pub(crate) mod tests { // A second spend fails because there are no usable notes assert_matches!( - create_spend_to_address( - &mut db_data, - &testing::network(), - test_prover(), + test.create_spend_to_address( &usk, &to, Amount::from_u64(2000).unwrap(), @@ -982,32 +832,17 @@ pub(crate) mod tests { // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 41 (that don't send us funds) // until just before the first transaction expires for i in 1..42 { - cb = fake_compact_block( - sapling_activation_height() + i, - cb.hash(), + test.generate_next_block( &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, - i, - ) - .0; - insert_into_cache(&db_cache, &cb); + ); } - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height() + 1, - 41, - ) - .unwrap(); + test.scan_cached_blocks(h1 + 1, 41); // Second spend still fails assert_matches!( - create_spend_to_address( - &mut db_data, - &testing::network(), - test_prover(), + test.create_spend_to_address( &usk, &to, Amount::from_u64(2000).unwrap(), @@ -1023,30 +858,15 @@ pub(crate) mod tests { ); // Mine block SAPLING_ACTIVATION_HEIGHT + 42 so that the first transaction expires - cb = fake_compact_block( - sapling_activation_height() + 42, - cb.hash(), + let (h43, _, _) = test.generate_next_block( &ExtendedSpendingKey::master(&[42]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, - 42, - ) - .0; - insert_into_cache(&db_cache, &cb); - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height() + 42, - 1, - ) - .unwrap(); + ); + test.scan_cached_blocks(h43, 1); // Second spend should now succeed - create_spend_to_address( - &mut db_data, - &testing::network(), - test_prover(), + test.create_spend_to_address( &usk, &to, Amount::from_u64(2000).unwrap(), @@ -1059,41 +879,19 @@ pub(crate) mod tests { #[test] fn ovk_policy_prevents_recovery_from_chain() { - let network = testing::network(); - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb(Connection::open(cache_file.path()).unwrap()); - init_cache_database(&db_cache).unwrap(); - - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), network).unwrap(); - init_wallet_db(&mut db_data, None).unwrap(); + let mut test = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = db_data.create_account(&seed).unwrap(); + let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); - let (mut cb, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), - &dfvk, - AddressType::DefaultExternal, - value, - 0, - ); - insert_into_cache(&db_cache, &cb); - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 1, - ) - .unwrap(); + let (h1, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + test.scan_cached_blocks(h1, 1); assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), value ); @@ -1102,7 +900,7 @@ pub(crate) mod tests { let to = addr2.into(); #[allow(clippy::type_complexity)] - let send_and_recover_with_policy = |db_data: &mut WalletDb, + let send_and_recover_with_policy = |test: &mut TestRunner, ovk_policy| -> Result< Option<(Note, PaymentAddress, MemoBytes)>, @@ -1114,10 +912,7 @@ pub(crate) mod tests { ReceivedNoteId, >, > { - let txid = create_spend_to_address( - db_data, - &testing::network(), - test_prover(), + let txid = test.create_spend_to_address( &usk, &to, Amount::from_u64(15000).unwrap(), @@ -1127,7 +922,8 @@ pub(crate) mod tests { )?; // Fetch the transaction from the database - let raw_tx: Vec<_> = db_data + let raw_tx: Vec<_> = test + .wallet() .conn .query_row( "SELECT raw FROM transactions @@ -1141,8 +937,8 @@ pub(crate) mod tests { for output in tx.sapling_bundle().unwrap().shielded_outputs() { // Find the output that decrypts with the external OVK let result = try_sapling_output_recovery( - &network, - sapling_activation_height(), + &network(), + h1, &dfvk.to_ovk(Scope::External), output, ); @@ -1158,96 +954,61 @@ pub(crate) mod tests { // Send some of the funds to another address, keeping history. // The recipient output is decryptable by the sender. assert_matches!( - send_and_recover_with_policy(&mut db_data, OvkPolicy::Sender), + send_and_recover_with_policy(&mut test, OvkPolicy::Sender), Ok(Some((_, recovered_to, _))) if recovered_to == addr2 ); // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 42 (that don't send us funds) // so that the first transaction expires for i in 1..=42 { - cb = fake_compact_block( - sapling_activation_height() + i, - cb.hash(), + test.generate_next_block( &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, - i, - ) - .0; - insert_into_cache(&db_cache, &cb); + ); } - scan_cached_blocks( - &network, - &db_cache, - &mut db_data, - sapling_activation_height() + 1, - 42, - ) - .unwrap(); + test.scan_cached_blocks(h1 + 1, 42); // Send the funds again, discarding history. // Neither transaction output is decryptable by the sender. assert_matches!( - send_and_recover_with_policy(&mut db_data, OvkPolicy::Discard), + send_and_recover_with_policy(&mut test, OvkPolicy::Discard), Ok(None) ); } #[test] fn create_to_address_succeeds_to_t_addr_zero_change() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb(Connection::open(cache_file.path()).unwrap()); - init_cache_database(&db_cache).unwrap(); - - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, None).unwrap(); + let mut test = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = db_data.create_account(&seed).unwrap(); + let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note let value = Amount::from_u64(60000).unwrap(); - let (cb, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), - &dfvk, - AddressType::DefaultExternal, - value, - 0, - ); - insert_into_cache(&db_cache, &cb); - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 1, - ) - .unwrap(); + let (h, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + test.scan_cached_blocks(h, 1); // Verified balance matches total balance - let (_, anchor_height) = db_data + let (_, anchor_height) = test + .wallet() .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) .unwrap() .unwrap(); assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), value ); assert_eq!( - get_balance_at(&db_data.conn, AccountId::from(0), anchor_height).unwrap(), + get_balance_at(&test.wallet().conn, AccountId::from(0), anchor_height).unwrap(), value ); let to = TransparentAddress::PublicKey([7; 20]).into(); assert_matches!( - create_spend_to_address( - &mut db_data, - &testing::network(), - test_prover(), + test.create_spend_to_address( &usk, &to, Amount::from_u64(50000).unwrap(), @@ -1261,59 +1022,36 @@ pub(crate) mod tests { #[test] fn create_to_address_spends_a_change_note() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb(Connection::open(cache_file.path()).unwrap()); - init_cache_database(&db_cache).unwrap(); - - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, None).unwrap(); + let mut test = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = db_data.create_account(&seed).unwrap(); + let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note let value = Amount::from_u64(60000).unwrap(); - let (cb, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), - &dfvk, - AddressType::Internal, - value, - 0, - ); - insert_into_cache(&db_cache, &cb); - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 1, - ) - .unwrap(); + let (h, _, _) = test.generate_next_block(&dfvk, AddressType::Internal, value); + test.scan_cached_blocks(h, 1); // Verified balance matches total balance - let (_, anchor_height) = db_data + let (_, anchor_height) = test + .wallet() .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) .unwrap() .unwrap(); assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), value ); assert_eq!( - get_balance_at(&db_data.conn, AccountId::from(0), anchor_height).unwrap(), + get_balance_at(&test.wallet().conn, AccountId::from(0), anchor_height).unwrap(), value ); let to = TransparentAddress::PublicKey([7; 20]).into(); assert_matches!( - create_spend_to_address( - &mut db_data, - &testing::network(), - test_prover(), + test.create_spend_to_address( &usk, &to, Amount::from_u64(50000).unwrap(), @@ -1327,65 +1065,44 @@ pub(crate) mod tests { #[test] fn zip317_spend() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb(Connection::open(cache_file.path()).unwrap()); - init_cache_database(&db_cache).unwrap(); - - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, None).unwrap(); + let mut test = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = db_data.create_account(&seed).unwrap(); + let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet - let (mut cb, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), + let (h1, _, _) = test.generate_next_block( &dfvk, AddressType::Internal, Amount::from_u64(50000).unwrap(), - 0, ); - insert_into_cache(&db_cache, &cb); // Add 10 dust notes to the wallet - for i in 1..=10 { - cb = fake_compact_block( - sapling_activation_height() + i, - cb.hash(), + for _ in 1..=10 { + test.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(1000).unwrap(), - i, - ) - .0; - insert_into_cache(&db_cache, &cb); + ); } - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 11, - ) - .unwrap(); + test.scan_cached_blocks(h1, 11); // Verified balance matches total balance let total = Amount::from_u64(60000).unwrap(); - let (_, anchor_height) = db_data + let (_, anchor_height) = test + .wallet() .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) .unwrap() .unwrap(); assert_eq!( - get_balance(&db_data.conn, AccountId::from(0)).unwrap(), + get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), total ); assert_eq!( - get_balance_at(&db_data.conn, AccountId::from(0), anchor_height).unwrap(), + get_balance_at(&test.wallet().conn, AccountId::from(0), anchor_height).unwrap(), total ); @@ -1406,10 +1123,7 @@ pub(crate) mod tests { .unwrap(); assert_matches!( - spend( - &mut db_data, - &testing::network(), - test_prover(), + test.spend( &input_selector, &usk, req, @@ -1434,10 +1148,7 @@ pub(crate) mod tests { .unwrap(); assert_matches!( - spend( - &mut db_data, - &testing::network(), - test_prover(), + test.spend( &input_selector, &usk, req, @@ -1451,32 +1162,38 @@ pub(crate) mod tests { #[test] #[cfg(feature = "transparent-inputs")] fn shield_transparent() { - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb(Connection::open(cache_file.path()).unwrap()); - init_cache_database(&db_cache).unwrap(); - - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, None).unwrap(); + let mut test = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (account_id, usk) = db_data.create_account(&seed).unwrap(); + let (account_id, usk) = test.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); - let uaddr = db_data.get_current_address(account_id).unwrap().unwrap(); + let uaddr = test + .wallet() + .get_current_address(account_id) + .unwrap() + .unwrap(); let taddr = uaddr.transparent().unwrap(); + // Ensure that the wallet has at least one block + let (h, _, _) = test.generate_next_block( + &dfvk, + AddressType::Internal, + Amount::from_u64(50000).unwrap(), + ); + test.scan_cached_blocks(h, 1); + let utxo = WalletTransparentOutput::from_parts( OutPoint::new([1u8; 32], 1), TxOut { value: Amount::from_u64(10000).unwrap(), script_pubkey: taddr.script(), }, - sapling_activation_height(), + h, ) .unwrap(); - let res0 = db_data.put_received_transparent_utxo(&utxo); + let res0 = test.wallet_mut().put_received_transparent_utxo(&utxo); assert!(matches!(res0, Ok(_))); let input_selector = GreedyInputSelector::new( @@ -1484,30 +1201,8 @@ pub(crate) mod tests { DustOutputPolicy::default(), ); - // Ensure that the wallet has at least one block - let (cb, _) = fake_compact_block( - sapling_activation_height(), - BlockHash([0; 32]), - &dfvk, - AddressType::Internal, - Amount::from_u64(50000).unwrap(), - 0, - ); - insert_into_cache(&db_cache, &cb); - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - sapling_activation_height(), - 1, - ) - .unwrap(); - assert_matches!( - shield_transparent_funds( - &mut db_data, - &testing::network(), - test_prover(), + test.shield_transparent_funds( &input_selector, NonNegativeAmount::from_u64(10000).unwrap(), &usk, diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 27863ff9e5..8bfb8c1cee 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -738,11 +738,9 @@ mod tests { use std::ops::Range; use incrementalmerkletree::{Hashable, Level}; - use rusqlite::Connection; use secrecy::Secret; - use tempfile::NamedTempFile; use zcash_client_backend::data_api::{ - chain::{scan_cached_blocks, CommitmentTreeRoot}, + chain::CommitmentTreeRoot, scanning::{ScanPriority, ScanRange}, WalletCommitmentTrees, WalletRead, WalletWrite, }; @@ -754,16 +752,8 @@ mod tests { }; use crate::{ - chain::init::init_cache_database, - testing::{ - self, fake_compact_block, init_test_accounts_table, insert_into_cache, - sapling_activation_height, AddressType, - }, - wallet::{ - init::{init_blocks_table, init_wallet_db}, - scanning::suggest_scan_ranges, - }, - BlockDb, WalletDb, + testing::{sapling_activation_height, AddressType, TestBuilder}, + wallet::{init::init_blocks_table, scanning::suggest_scan_ranges}, }; use super::{join_nonoverlapping, Joined, RangeOrdering, SpanningTree}; @@ -1093,21 +1083,18 @@ mod tests { fn scan_complete() { use ScanPriority::*; - let cache_file = NamedTempFile::new().unwrap(); - let db_cache = BlockDb(Connection::open(cache_file.path()).unwrap()); - init_cache_database(&db_cache).unwrap(); + let mut test = TestBuilder::new() + .with_block_cache() + .with_seed(Secret::new(vec![])) + .with_test_account() + .build(); - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); - - // Add an account to the wallet. - let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); + let dfvk = test.test_account_sapling().unwrap(); assert_matches!( // In the following, we don't care what the root hashes are, they just need to be // distinct. - db_data.put_sapling_subtree_roots( + test.wallet_mut().put_sapling_subtree_roots( 0, &[ CommitmentTreeRoot::from_parts( @@ -1134,7 +1121,7 @@ mod tests { let initial_height = sapling_activation_height() + 310; let value = Amount::from_u64(50000).unwrap(); - let (mut cb, _) = fake_compact_block( + test.generate_block_at( initial_height, BlockHash([0; 32]), &dfvk, @@ -1142,36 +1129,21 @@ mod tests { value, initial_sapling_tree_size, ); - insert_into_cache(&db_cache, &cb); - for i in 1..=10 { - cb = fake_compact_block( - initial_height + i, - cb.hash(), + for _ in 1..=10 { + test.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(10000).unwrap(), - initial_sapling_tree_size + i, - ) - .0; - insert_into_cache(&db_cache, &cb); + ); } - assert_matches!( - scan_cached_blocks( - &testing::network(), - &db_cache, - &mut db_data, - initial_height, - 10, - ), - Ok(()) - ); + test.scan_cached_blocks(initial_height, 10); // Verify the that adjacent range needed to make the note spendable has been prioritized. let sap_active = u32::from(sapling_activation_height()); assert_matches!( - db_data.suggest_scan_ranges(), + test.wallet().suggest_scan_ranges(), Ok(scan_ranges) if scan_ranges == vec![ scan_range((sap_active + 300)..(sap_active + 310), FoundNote) ] @@ -1179,7 +1151,7 @@ mod tests { // Check that the scanned range has been properly persisted. assert_matches!( - suggest_scan_ranges(&db_data.conn, Scanned), + suggest_scan_ranges(&test.wallet().conn, Scanned), Ok(scan_ranges) if scan_ranges == vec![ scan_range((sap_active + 300)..(sap_active + 310), FoundNote), scan_range((sap_active + 310)..(sap_active + 320), Scanned) @@ -1189,14 +1161,15 @@ mod tests { // Simulate the wallet going offline for a bit, update the chain tip to 20 blocks in the // future. assert_matches!( - db_data.update_chain_tip(sapling_activation_height() + 340), + test.wallet_mut() + .update_chain_tip(sapling_activation_height() + 340), Ok(()) ); // Check the scan range again, we should see a `ChainTip` range for the period we've been // offline. assert_matches!( - db_data.suggest_scan_ranges(), + test.wallet().suggest_scan_ranges(), Ok(scan_ranges) if scan_ranges == vec![ scan_range((sap_active + 320)..(sap_active + 341), ChainTip), scan_range((sap_active + 300)..(sap_active + 310), ChainTip) @@ -1205,14 +1178,15 @@ mod tests { // Now simulate a jump ahead more than 100 blocks. assert_matches!( - db_data.update_chain_tip(sapling_activation_height() + 450), + test.wallet_mut() + .update_chain_tip(sapling_activation_height() + 450), Ok(()) ); // Check the scan range again, we should see a `Validate` range for the previous wallet // tip, and then a `ChainTip` for the remaining range. assert_matches!( - db_data.suggest_scan_ranges(), + test.wallet().suggest_scan_ranges(), Ok(scan_ranges) if scan_ranges == vec![ scan_range((sap_active + 319)..(sap_active + 329), Verify), scan_range((sap_active + 329)..(sap_active + 451), ChainTip), @@ -1225,22 +1199,22 @@ mod tests { fn init_blocks_table_creates_ignored_range() { use ScanPriority::*; - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); + let mut test = TestBuilder::new().with_seed(Secret::new(vec![])).build(); - let sap_active = db_data + let sap_active = test + .wallet() .params .activation_height(NetworkUpgrade::Sapling) .unwrap(); // Initialise the blocks table. We use Canopy activation as an arbitrary birthday height // that's greater than Sapling activation. - let birthday_height = db_data + let birthday_height = test + .wallet() .params .activation_height(NetworkUpgrade::Canopy) .unwrap(); init_blocks_table( - &mut db_data, + test.wallet_mut(), birthday_height, BlockHash([1; 32]), 1, @@ -1256,12 +1230,12 @@ mod tests { ), ]; assert_matches!( - suggest_scan_ranges(&db_data.conn, Ignored), + suggest_scan_ranges(&test.wallet().conn, Ignored), Ok(scan_ranges) if scan_ranges == expected ); // Set up some shard history - db_data + test.wallet_mut() .put_sapling_subtree_roots( 0, &[ @@ -1280,11 +1254,12 @@ mod tests { .unwrap(); // Update the chain tip - let tip_height = db_data + let tip_height = test + .wallet() .params .activation_height(NetworkUpgrade::Nu5) .unwrap(); - db_data.update_chain_tip(tip_height).unwrap(); + test.wallet_mut().update_chain_tip(tip_height).unwrap(); // Verify that the suggested scan ranges match what is expected let expected = vec![ @@ -1315,7 +1290,7 @@ mod tests { ]; assert_matches!( - suggest_scan_ranges(&db_data.conn, Ignored), + suggest_scan_ranges(&test.wallet().conn, Ignored), Ok(scan_ranges) if scan_ranges == expected ); } From 578eac15df42190a1e04fd573c379ca685a12b60 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 28 Aug 2023 18:54:35 +0000 Subject: [PATCH 0112/1122] zcash_client_sqlite: Rename `TestRunner` to `TestState` --- zcash_client_sqlite/src/chain.rs | 132 +++++++------- zcash_client_sqlite/src/lib.rs | 49 +++--- zcash_client_sqlite/src/testing.rs | 14 +- zcash_client_sqlite/src/wallet.rs | 34 ++-- zcash_client_sqlite/src/wallet/sapling.rs | 192 ++++++++++----------- zcash_client_sqlite/src/wallet/scanning.rs | 42 ++--- 6 files changed, 230 insertions(+), 233 deletions(-) diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 35f90a80a7..519f891bb2 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -352,66 +352,66 @@ mod tests { #[test] fn valid_chain_states() { - let mut test = TestBuilder::new() + let mut st = TestBuilder::new() .with_block_cache() .with_seed(Secret::new(vec![])) .with_test_account() .build(); - let dfvk = test.test_account_sapling().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Empty chain should return None - assert_matches!(test.wallet().chain_height(), Ok(None)); + assert_matches!(st.wallet().chain_height(), Ok(None)); // Create a fake CompactBlock sending value to the address - let (h1, _, _) = test.generate_next_block( + let (h1, _, _) = st.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), ); // Scan the cache - test.scan_cached_blocks(h1, 1); + st.scan_cached_blocks(h1, 1); // Create a second fake CompactBlock sending more value to the address - let (h2, _, _) = test.generate_next_block( + let (h2, _, _) = st.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(7).unwrap(), ); // Scanning should detect no inconsistencies - test.scan_cached_blocks(h2, 1); + st.scan_cached_blocks(h2, 1); } #[test] fn invalid_chain_cache_disconnected() { - let mut test = TestBuilder::new() + let mut st = TestBuilder::new() .with_block_cache() .with_seed(Secret::new(vec![])) .with_test_account() .build(); - let dfvk = test.test_account_sapling().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Create some fake CompactBlocks - let (h, _, _) = test.generate_next_block( + let (h, _, _) = st.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), ); - let (last_contiguous_height, _, _) = test.generate_next_block( + let (last_contiguous_height, _, _) = st.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(7).unwrap(), ); // Scanning the cache should find no inconsistencies - test.scan_cached_blocks(h, 2); + st.scan_cached_blocks(h, 2); // Create more fake CompactBlocks that don't connect to the scanned ones let disconnect_height = last_contiguous_height + 1; - test.generate_block_at( + st.generate_block_at( disconnect_height, BlockHash([1; 32]), &dfvk, @@ -419,7 +419,7 @@ mod tests { Amount::from_u64(8).unwrap(), 2, ); - test.generate_next_block( + st.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(3).unwrap(), @@ -427,7 +427,7 @@ mod tests { // Data+cache chain should be invalid at the data/cache boundary assert_matches!( - test.try_scan_cached_blocks( + st.try_scan_cached_blocks( disconnect_height, 2 ), @@ -438,99 +438,99 @@ mod tests { #[test] fn data_db_truncation() { - let mut test = TestBuilder::new() + let mut st = TestBuilder::new() .with_block_cache() .with_seed(Secret::new(vec![])) .with_test_account() .build(); - let dfvk = test.test_account_sapling().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Account balance should be zero assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); // Create fake CompactBlocks sending value to the address let value = Amount::from_u64(5).unwrap(); let value2 = Amount::from_u64(7).unwrap(); - let (h, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - test.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); // Scan the cache - test.scan_cached_blocks(h, 2); + st.scan_cached_blocks(h, 2); // Account balance should reflect both received notes assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), (value + value2).unwrap() ); // "Rewind" to height of last scanned block - test.wallet_mut() + st.wallet_mut() .transactionally(|wdb| truncate_to_height(wdb.conn.0, &wdb.params, h + 1)) .unwrap(); // Account balance should be unaltered assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), (value + value2).unwrap() ); // Rewind so that one block is dropped - test.wallet_mut() + st.wallet_mut() .transactionally(|wdb| truncate_to_height(wdb.conn.0, &wdb.params, h)) .unwrap(); // Account balance should only contain the first received note assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), value ); // Scan the cache again - test.scan_cached_blocks(h, 2); + st.scan_cached_blocks(h, 2); // Account balance should again reflect both received notes assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), (value + value2).unwrap() ); } #[test] fn scan_cached_blocks_allows_blocks_out_of_order() { - let mut test = TestBuilder::new() + let mut st = TestBuilder::new() .with_block_cache() .with_seed(Secret::new(vec![])) .build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); + let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Create a block with height SAPLING_ACTIVATION_HEIGHT let value = Amount::from_u64(50000).unwrap(); - let (h1, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - test.scan_cached_blocks(h1, 1); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h1, 1); assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), value ); // Create blocks to reach SAPLING_ACTIVATION_HEIGHT + 2 - let (h2, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - let (h3, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let (h3, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); // Scan the later block first - test.scan_cached_blocks(h3, 1); + st.scan_cached_blocks(h3, 1); // Now scan the block of height SAPLING_ACTIVATION_HEIGHT + 1 - test.scan_cached_blocks(h2, 1); + st.scan_cached_blocks(h2, 1); assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), Amount::from_u64(150_000).unwrap() ); @@ -549,7 +549,7 @@ mod tests { DustOutputPolicy::default(), ); assert_matches!( - test.spend( + st.spend( &input_selector, &usk, req, @@ -562,74 +562,74 @@ mod tests { #[test] fn scan_cached_blocks_finds_received_notes() { - let mut test = TestBuilder::new() + let mut st = TestBuilder::new() .with_block_cache() .with_seed(Secret::new(vec![])) .with_test_account() .build(); - let dfvk = test.test_account_sapling().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Account balance should be zero assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); // Create a fake CompactBlock sending value to the address let value = Amount::from_u64(5).unwrap(); - let (h1, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); // Scan the cache - test.scan_cached_blocks(h1, 1); + st.scan_cached_blocks(h1, 1); // Account balance should reflect the received note assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), value ); // Create a second fake CompactBlock sending more value to the address let value2 = Amount::from_u64(7).unwrap(); - let (h2, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); + let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); // Scan the cache again - test.scan_cached_blocks(h2, 1); + st.scan_cached_blocks(h2, 1); // Account balance should reflect both received notes assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), (value + value2).unwrap() ); } #[test] fn scan_cached_blocks_finds_change_notes() { - let mut test = TestBuilder::new() + let mut st = TestBuilder::new() .with_block_cache() .with_seed(Secret::new(vec![])) .with_test_account() .build(); - let dfvk = test.test_account_sapling().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Account balance should be zero assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); // Create a fake CompactBlock sending value to the address let value = Amount::from_u64(5).unwrap(); let (received_height, _, nf) = - test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); // Scan the cache - test.scan_cached_blocks(received_height, 1); + st.scan_cached_blocks(received_height, 1); // Account balance should reflect the received note assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), value ); @@ -637,60 +637,60 @@ mod tests { let extsk2 = ExtendedSpendingKey::master(&[0]); let to2 = extsk2.default_address().1; let value2 = Amount::from_u64(2).unwrap(); - let (spent_height, _) = test.generate_next_block_spending(&dfvk, (nf, value), to2, value2); + let (spent_height, _) = st.generate_next_block_spending(&dfvk, (nf, value), to2, value2); // Scan the cache again - test.scan_cached_blocks(spent_height, 1); + st.scan_cached_blocks(spent_height, 1); // Account balance should equal the change assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), (value - value2).unwrap() ); } #[test] fn scan_cached_blocks_detects_spends_out_of_order() { - let mut test = TestBuilder::new() + let mut st = TestBuilder::new() .with_block_cache() .with_seed(Secret::new(vec![])) .with_test_account() .build(); - let dfvk = test.test_account_sapling().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Account balance should be zero assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); // Create a fake CompactBlock sending value to the address let value = Amount::from_u64(5).unwrap(); let (received_height, _, nf) = - test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); // Create a second fake CompactBlock spending value from the address let extsk2 = ExtendedSpendingKey::master(&[0]); let to2 = extsk2.default_address().1; let value2 = Amount::from_u64(2).unwrap(); - let (spent_height, _) = test.generate_next_block_spending(&dfvk, (nf, value), to2, value2); + let (spent_height, _) = st.generate_next_block_spending(&dfvk, (nf, value), to2, value2); // Scan the spending block first. - test.scan_cached_blocks(spent_height, 1); + st.scan_cached_blocks(spent_height, 1); // Account balance should equal the change assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), (value - value2).unwrap() ); // Now scan the block in which we received the note that was spent. - test.scan_cached_blocks(received_height, 1); + st.scan_cached_blocks(received_height, 1); // Account balance should be the same. assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), (value - value2).unwrap() ); } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index e3ecf3ecac..dde55aa635 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -1100,20 +1100,17 @@ mod tests { #[test] pub(crate) fn get_next_available_address() { - let mut test = TestBuilder::new().with_test_account().build(); + let mut st = TestBuilder::new().with_test_account().build(); let account = AccountId::from(0); - let current_addr = test.wallet().get_current_address(account).unwrap(); + let current_addr = st.wallet().get_current_address(account).unwrap(); assert!(current_addr.is_some()); - let addr2 = test - .wallet_mut() - .get_next_available_address(account) - .unwrap(); + let addr2 = st.wallet_mut().get_next_available_address(account).unwrap(); assert!(addr2.is_some()); assert_ne!(current_addr, addr2); - let addr2_cur = test.wallet().get_current_address(account).unwrap(); + let addr2_cur = st.wallet().get_current_address(account).unwrap(); assert_eq!(addr2, addr2_cur); } @@ -1122,17 +1119,17 @@ mod tests { fn transparent_receivers() { use secrecy::Secret; - let test = TestBuilder::new() + let st = TestBuilder::new() .with_block_cache() .with_seed(Secret::new(vec![])) .with_test_account() .build(); // Add an account to the wallet. - let (ufvk, taddr) = test.test_account().unwrap(); + let (ufvk, taddr) = st.test_account().unwrap(); let taddr = taddr.unwrap(); - let receivers = test.wallet().get_transparent_receivers(0.into()).unwrap(); + let receivers = st.wallet().get_transparent_receivers(0.into()).unwrap(); // The receiver for the default UA should be in the set. assert!(receivers.contains_key(ufvk.default_address().0.transparent().unwrap())); @@ -1144,44 +1141,44 @@ mod tests { #[cfg(feature = "unstable")] #[test] pub(crate) fn fsblockdb_api() { - let mut test = TestBuilder::new().with_fs_block_cache().build(); + let mut st = TestBuilder::new().with_fs_block_cache().build(); // The BlockMeta DB starts off empty. - assert_eq!(test.cache().get_max_cached_height().unwrap(), None); + assert_eq!(st.cache().get_max_cached_height().unwrap(), None); // Generate some fake CompactBlocks. let seed = [0u8; 32]; let account = AccountId::from(0); - let extsk = sapling::spending_key(&seed, test.wallet().params.coin_type(), account); + let extsk = sapling::spending_key(&seed, st.wallet().params.coin_type(), account); let dfvk = extsk.to_diversifiable_full_viewing_key(); - let (h1, meta1, _) = test.generate_next_block( + let (h1, meta1, _) = st.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), ); - let (h2, meta2, _) = test.generate_next_block( + let (h2, meta2, _) = st.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(10).unwrap(), ); // The BlockMeta DB is not updated until we do so explicitly. - assert_eq!(test.cache().get_max_cached_height().unwrap(), None); + assert_eq!(st.cache().get_max_cached_height().unwrap(), None); // Inform the BlockMeta DB about the newly-persisted CompactBlocks. - test.cache().write_block_metadata(&[meta1, meta2]).unwrap(); + st.cache().write_block_metadata(&[meta1, meta2]).unwrap(); // The BlockMeta DB now sees blocks up to height 2. - assert_eq!(test.cache().get_max_cached_height().unwrap(), Some(h2),); - assert_eq!(test.cache().find_block(h1).unwrap(), Some(meta1)); - assert_eq!(test.cache().find_block(h2).unwrap(), Some(meta2)); - assert_eq!(test.cache().find_block(h2 + 1).unwrap(), None); + assert_eq!(st.cache().get_max_cached_height().unwrap(), Some(h2),); + assert_eq!(st.cache().find_block(h1).unwrap(), Some(meta1)); + assert_eq!(st.cache().find_block(h2).unwrap(), Some(meta2)); + assert_eq!(st.cache().find_block(h2 + 1).unwrap(), None); // Rewinding to height 1 should cause the metadata for height 2 to be deleted. - test.cache().truncate_to_height(h1).unwrap(); - assert_eq!(test.cache().get_max_cached_height().unwrap(), Some(h1)); - assert_eq!(test.cache().find_block(h1).unwrap(), Some(meta1)); - assert_eq!(test.cache().find_block(h2).unwrap(), None); - assert_eq!(test.cache().find_block(h2 + 1).unwrap(), None); + st.cache().truncate_to_height(h1).unwrap(); + assert_eq!(st.cache().get_max_cached_height().unwrap(), Some(h1)); + assert_eq!(st.cache().find_block(h1).unwrap(), Some(meta1)); + assert_eq!(st.cache().find_block(h2).unwrap(), None); + assert_eq!(st.cache().find_block(h2 + 1).unwrap(), None); } } diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index db6a135fbc..6c4e539421 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -129,8 +129,8 @@ impl TestBuilder { self } - /// Builds the runner for this test. - pub(crate) fn build(self) -> TestRunner { + /// Builds the state for this test. + pub(crate) fn build(self) -> TestState { let params = network(); let data_file = NamedTempFile::new().unwrap(); @@ -144,7 +144,7 @@ impl TestBuilder { None }; - TestRunner { + TestState { params, cache: self.cache, latest_cached_block: None, @@ -155,8 +155,8 @@ impl TestBuilder { } } -/// A `zcash_client_sqlite` test runner. -pub(crate) struct TestRunner { +/// The state for a `zcash_client_sqlite` test. +pub(crate) struct TestState { params: Network, cache: Cache, latest_cached_block: Option<(BlockHeight, BlockHash, u32)>, @@ -165,7 +165,7 @@ pub(crate) struct TestRunner { test_account: Option<(UnifiedFullViewingKey, Option)>, } -impl TestRunner +impl TestState where ::Error: fmt::Debug, { @@ -297,7 +297,7 @@ where } } -impl TestRunner { +impl TestState { /// Exposes an immutable reference to the test's [`WalletDb`]. pub(crate) fn wallet(&self) -> &WalletDb { &self.db_data diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index d2fdbc639f..0434c2c6b6 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1594,20 +1594,20 @@ mod tests { #[test] fn empty_database_has_no_balance() { - let test = TestBuilder::new() + let st = TestBuilder::new() .with_seed(Secret::new(vec![])) .with_test_account() .build(); // The account should be empty assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); // We can't get an anchor height, as we have not scanned any blocks. assert_eq!( - test.wallet() + st.wallet() .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) .unwrap(), None @@ -1615,11 +1615,11 @@ mod tests { // An invalid account has zero balance assert_matches!( - test.wallet().get_current_address(AccountId::from(1)), + st.wallet().get_current_address(AccountId::from(1)), Ok(None) ); assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); } @@ -1627,19 +1627,19 @@ mod tests { #[test] #[cfg(feature = "transparent-inputs")] fn put_received_transparent_utxo() { - let mut test = TestBuilder::new().build(); + let mut st = TestBuilder::new().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (account_id, _usk) = test.wallet_mut().create_account(&seed).unwrap(); - let uaddr = test + let (account_id, _usk) = st.wallet_mut().create_account(&seed).unwrap(); + let uaddr = st .wallet() .get_current_address(account_id) .unwrap() .unwrap(); let taddr = uaddr.transparent().unwrap(); - let bal_absent = test + let bal_absent = st .wallet() .get_transparent_balances(account_id, BlockHeight::from_u32(12345)) .unwrap(); @@ -1655,7 +1655,7 @@ mod tests { ) .unwrap(); - let res0 = test.wallet_mut().put_received_transparent_utxo(&utxo); + let res0 = st.wallet_mut().put_received_transparent_utxo(&utxo); assert_matches!(res0, Ok(_)); // Change the mined height of the UTXO and upsert; we should get back @@ -1669,11 +1669,11 @@ mod tests { BlockHeight::from_u32(34567), ) .unwrap(); - let res1 = test.wallet_mut().put_received_transparent_utxo(&utxo2); + let res1 = st.wallet_mut().put_received_transparent_utxo(&utxo2); assert_matches!(res1, Ok(id) if id == res0.unwrap()); assert_matches!( - test.wallet().get_unspent_transparent_outputs( + st.wallet().get_unspent_transparent_outputs( taddr, BlockHeight::from_u32(12345), &[] @@ -1682,7 +1682,7 @@ mod tests { ); assert_matches!( - test.wallet().get_unspent_transparent_outputs( + st.wallet().get_unspent_transparent_outputs( taddr, BlockHeight::from_u32(34567), &[] @@ -1694,21 +1694,21 @@ mod tests { ); assert_matches!( - test.wallet().get_transparent_balances(account_id, BlockHeight::from_u32(34567)), + st.wallet().get_transparent_balances(account_id, BlockHeight::from_u32(34567)), Ok(h) if h.get(taddr) == Amount::from_u64(100000).ok().as_ref() ); // Artificially delete the address from the addresses table so that // we can ensure the update fails if the join doesn't work. - test.wallet() + st.wallet() .conn .execute( "DELETE FROM addresses WHERE cached_transparent_receiver_address = ?", - [Some(taddr.encode(&test.wallet().params))], + [Some(taddr.encode(&st.wallet().params))], ) .unwrap(); - let res2 = test.wallet_mut().put_received_transparent_utxo(&utxo2); + let res2 = st.wallet_mut().put_received_transparent_utxo(&utxo2); assert_matches!(res2, Err(_)); } } diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 8d95fcaea4..edf4ac2056 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -439,7 +439,7 @@ pub(crate) mod tests { use crate::{ error::SqliteClientError, - testing::{self, network, AddressType, BlockCache, TestBuilder, TestRunner}, + testing::{self, network, AddressType, BlockCache, TestBuilder, TestState}, wallet::{commitment_tree, get_balance, get_balance_at}, AccountId, NoteId, ReceivedNoteId, }; @@ -461,30 +461,30 @@ pub(crate) mod tests { #[test] fn send_proposed_transfer() { - let mut test = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (account, usk) = test.wallet_mut().create_account(&seed).unwrap(); + let (account, usk) = st.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note let value = Amount::from_u64(60000).unwrap(); - let (h, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - test.scan_cached_blocks(h, 1); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h, 1); // Verified balance matches total balance - let (_, anchor_height) = test + let (_, anchor_height) = st .wallet() .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) .unwrap() .unwrap(); assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), value ); assert_eq!( - get_balance_at(&test.wallet().conn, AccountId::from(0), anchor_height).unwrap(), + get_balance_at(&st.wallet().conn, AccountId::from(0), anchor_height).unwrap(), value ); @@ -504,7 +504,7 @@ pub(crate) mod tests { let change_strategy = fixed::SingleOutputChangeStrategy::new(fee_rule); let input_selector = &GreedyInputSelector::new(change_strategy, DustOutputPolicy::default()); - let proposal_result = test.propose_transfer( + let proposal_result = st.propose_transfer( account, input_selector, request, @@ -513,7 +513,7 @@ pub(crate) mod tests { assert_matches!(proposal_result, Ok(_)); let change_memo = "Test change memo".parse::().unwrap(); - let create_proposed_result = test.create_proposed_transaction( + let create_proposed_result = st.create_proposed_transaction( &usk, OvkPolicy::Sender, proposal_result.unwrap(), @@ -525,7 +525,7 @@ pub(crate) mod tests { let sent_tx_id = create_proposed_result.unwrap(); // Verify that the sent transaction was stored and that we can decrypt the memos - let tx = test + let tx = st .wallet() .get_transaction(sent_tx_id) .expect("Created transaction was stored."); @@ -549,7 +549,7 @@ pub(crate) mod tests { assert!(found_tx_empty_memo); // Verify that the stored sent notes match what we're expecting - let mut stmt_sent_notes = test + let mut stmt_sent_notes = st .wallet() .conn .prepare( @@ -580,7 +580,7 @@ pub(crate) mod tests { let mut found_sent_change_memo = false; let mut found_sent_empty_memo = false; for sent_note_id in sent_note_ids { - match test + match st .wallet() .get_memo(sent_note_id) .expect("Note id is valid") @@ -601,7 +601,7 @@ pub(crate) mod tests { // Check that querying for a nonexistent sent note returns None assert_matches!( - test.wallet() + st.wallet() .get_memo(NoteId::new(sent_tx_id, ShieldedProtocol::Sapling, 12345)), Ok(None) ); @@ -609,11 +609,11 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_on_incorrect_usk() { - let mut test = TestBuilder::new().with_seed(Secret::new(vec![])).build(); + let mut st = TestBuilder::new().with_seed(Secret::new(vec![])).build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); + let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); let to = dfvk.default_address().1.into(); @@ -623,7 +623,7 @@ pub(crate) mod tests { // Attempting to spend with a USK that is not in the wallet results in an error assert_matches!( - test.create_spend_to_address( + st.create_spend_to_address( &usk1, &to, Amount::from_u64(1).unwrap(), @@ -637,23 +637,23 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_with_no_blocks() { - let mut test = TestBuilder::new().build(); + let mut st = TestBuilder::new().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); + let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); let to = dfvk.default_address().1.into(); // Account balance should be zero assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), Amount::zero() ); // We cannot do anything if we aren't synchronised assert_matches!( - test.create_spend_to_address( + st.create_spend_to_address( &usk, &to, Amount::from_u64(1).unwrap(), @@ -667,49 +667,49 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_on_unverified_notes() { - let mut test = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); + let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); - let (h1, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - test.scan_cached_blocks(h1, 1); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h1, 1); // Verified balance matches total balance - let (_, anchor_height) = test + let (_, anchor_height) = st .wallet() .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) .unwrap() .unwrap(); assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), value ); assert_eq!( - get_balance_at(&test.wallet().conn, AccountId::from(0), anchor_height).unwrap(), + get_balance_at(&st.wallet().conn, AccountId::from(0), anchor_height).unwrap(), value ); // Add more funds to the wallet in a second note - let (h2, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - test.scan_cached_blocks(h2, 1); + let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h2, 1); // Verified balance does not include the second note - let (_, anchor_height2) = test + let (_, anchor_height2) = st .wallet() .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) .unwrap() .unwrap(); assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), (value + value).unwrap() ); assert_eq!( - get_balance_at(&test.wallet().conn, AccountId::from(0), anchor_height2).unwrap(), + get_balance_at(&st.wallet().conn, AccountId::from(0), anchor_height2).unwrap(), value ); @@ -717,7 +717,7 @@ pub(crate) mod tests { let extsk2 = ExtendedSpendingKey::master(&[]); let to = extsk2.default_address().1.into(); assert_matches!( - test.create_spend_to_address( + st.create_spend_to_address( &usk, &to, Amount::from_u64(70000).unwrap(), @@ -736,13 +736,13 @@ pub(crate) mod tests { // Mine blocks SAPLING_ACTIVATION_HEIGHT + 2 to 9 until just before the second // note is verified for _ in 2..10 { - test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); } - test.scan_cached_blocks(h2 + 1, 8); + st.scan_cached_blocks(h2 + 1, 8); // Second spend still fails assert_matches!( - test.create_spend_to_address( + st.create_spend_to_address( &usk, &to, Amount::from_u64(70000).unwrap(), @@ -759,12 +759,12 @@ pub(crate) mod tests { ); // Mine block 11 so that the second note becomes verified - let (h11, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - test.scan_cached_blocks(h11, 1); + let (h11, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h11, 1); // Second spend should now succeed assert_matches!( - test.create_spend_to_address( + st.create_spend_to_address( &usk, &to, Amount::from_u64(70000).unwrap(), @@ -778,22 +778,22 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_on_locked_notes() { - let mut test = TestBuilder::new() + let mut st = TestBuilder::new() .with_block_cache() .with_seed(Secret::new(vec![])) .build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); + let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); - let (h1, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - test.scan_cached_blocks(h1, 1); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h1, 1); assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), value ); @@ -801,7 +801,7 @@ pub(crate) mod tests { let extsk2 = ExtendedSpendingKey::master(&[]); let to = extsk2.default_address().1.into(); assert_matches!( - test.create_spend_to_address( + st.create_spend_to_address( &usk, &to, Amount::from_u64(15000).unwrap(), @@ -814,7 +814,7 @@ pub(crate) mod tests { // A second spend fails because there are no usable notes assert_matches!( - test.create_spend_to_address( + st.create_spend_to_address( &usk, &to, Amount::from_u64(2000).unwrap(), @@ -832,17 +832,17 @@ pub(crate) mod tests { // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 41 (that don't send us funds) // until just before the first transaction expires for i in 1..42 { - test.generate_next_block( + st.generate_next_block( &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, ); } - test.scan_cached_blocks(h1 + 1, 41); + st.scan_cached_blocks(h1 + 1, 41); // Second spend still fails assert_matches!( - test.create_spend_to_address( + st.create_spend_to_address( &usk, &to, Amount::from_u64(2000).unwrap(), @@ -858,15 +858,15 @@ pub(crate) mod tests { ); // Mine block SAPLING_ACTIVATION_HEIGHT + 42 so that the first transaction expires - let (h43, _, _) = test.generate_next_block( + let (h43, _, _) = st.generate_next_block( &ExtendedSpendingKey::master(&[42]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, ); - test.scan_cached_blocks(h43, 1); + st.scan_cached_blocks(h43, 1); // Second spend should now succeed - test.create_spend_to_address( + st.create_spend_to_address( &usk, &to, Amount::from_u64(2000).unwrap(), @@ -879,19 +879,19 @@ pub(crate) mod tests { #[test] fn ovk_policy_prevents_recovery_from_chain() { - let mut test = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); + let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); - let (h1, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - test.scan_cached_blocks(h1, 1); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h1, 1); assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), value ); @@ -900,7 +900,7 @@ pub(crate) mod tests { let to = addr2.into(); #[allow(clippy::type_complexity)] - let send_and_recover_with_policy = |test: &mut TestRunner, + let send_and_recover_with_policy = |test: &mut TestState, ovk_policy| -> Result< Option<(Note, PaymentAddress, MemoBytes)>, @@ -954,61 +954,61 @@ pub(crate) mod tests { // Send some of the funds to another address, keeping history. // The recipient output is decryptable by the sender. assert_matches!( - send_and_recover_with_policy(&mut test, OvkPolicy::Sender), + send_and_recover_with_policy(&mut st, OvkPolicy::Sender), Ok(Some((_, recovered_to, _))) if recovered_to == addr2 ); // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 42 (that don't send us funds) // so that the first transaction expires for i in 1..=42 { - test.generate_next_block( + st.generate_next_block( &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, ); } - test.scan_cached_blocks(h1 + 1, 42); + st.scan_cached_blocks(h1 + 1, 42); // Send the funds again, discarding history. // Neither transaction output is decryptable by the sender. assert_matches!( - send_and_recover_with_policy(&mut test, OvkPolicy::Discard), + send_and_recover_with_policy(&mut st, OvkPolicy::Discard), Ok(None) ); } #[test] fn create_to_address_succeeds_to_t_addr_zero_change() { - let mut test = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); + let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note let value = Amount::from_u64(60000).unwrap(); - let (h, _, _) = test.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - test.scan_cached_blocks(h, 1); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h, 1); // Verified balance matches total balance - let (_, anchor_height) = test + let (_, anchor_height) = st .wallet() .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) .unwrap() .unwrap(); assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), value ); assert_eq!( - get_balance_at(&test.wallet().conn, AccountId::from(0), anchor_height).unwrap(), + get_balance_at(&st.wallet().conn, AccountId::from(0), anchor_height).unwrap(), value ); let to = TransparentAddress::PublicKey([7; 20]).into(); assert_matches!( - test.create_spend_to_address( + st.create_spend_to_address( &usk, &to, Amount::from_u64(50000).unwrap(), @@ -1022,36 +1022,36 @@ pub(crate) mod tests { #[test] fn create_to_address_spends_a_change_note() { - let mut test = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); + let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet in a single note let value = Amount::from_u64(60000).unwrap(); - let (h, _, _) = test.generate_next_block(&dfvk, AddressType::Internal, value); - test.scan_cached_blocks(h, 1); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::Internal, value); + st.scan_cached_blocks(h, 1); // Verified balance matches total balance - let (_, anchor_height) = test + let (_, anchor_height) = st .wallet() .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) .unwrap() .unwrap(); assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), value ); assert_eq!( - get_balance_at(&test.wallet().conn, AccountId::from(0), anchor_height).unwrap(), + get_balance_at(&st.wallet().conn, AccountId::from(0), anchor_height).unwrap(), value ); let to = TransparentAddress::PublicKey([7; 20]).into(); assert_matches!( - test.create_spend_to_address( + st.create_spend_to_address( &usk, &to, Amount::from_u64(50000).unwrap(), @@ -1065,15 +1065,15 @@ pub(crate) mod tests { #[test] fn zip317_spend() { - let mut test = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = test.wallet_mut().create_account(&seed).unwrap(); + let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet - let (h1, _, _) = test.generate_next_block( + let (h1, _, _) = st.generate_next_block( &dfvk, AddressType::Internal, Amount::from_u64(50000).unwrap(), @@ -1081,28 +1081,28 @@ pub(crate) mod tests { // Add 10 dust notes to the wallet for _ in 1..=10 { - test.generate_next_block( + st.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(1000).unwrap(), ); } - test.scan_cached_blocks(h1, 11); + st.scan_cached_blocks(h1, 11); // Verified balance matches total balance let total = Amount::from_u64(60000).unwrap(); - let (_, anchor_height) = test + let (_, anchor_height) = st .wallet() .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) .unwrap() .unwrap(); assert_eq!( - get_balance(&test.wallet().conn, AccountId::from(0)).unwrap(), + get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), total ); assert_eq!( - get_balance_at(&test.wallet().conn, AccountId::from(0), anchor_height).unwrap(), + get_balance_at(&st.wallet().conn, AccountId::from(0), anchor_height).unwrap(), total ); @@ -1123,7 +1123,7 @@ pub(crate) mod tests { .unwrap(); assert_matches!( - test.spend( + st.spend( &input_selector, &usk, req, @@ -1148,7 +1148,7 @@ pub(crate) mod tests { .unwrap(); assert_matches!( - test.spend( + st.spend( &input_selector, &usk, req, @@ -1162,13 +1162,13 @@ pub(crate) mod tests { #[test] #[cfg(feature = "transparent-inputs")] fn shield_transparent() { - let mut test = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (account_id, usk) = test.wallet_mut().create_account(&seed).unwrap(); + let (account_id, usk) = st.wallet_mut().create_account(&seed).unwrap(); let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); - let uaddr = test + let uaddr = st .wallet() .get_current_address(account_id) .unwrap() @@ -1176,12 +1176,12 @@ pub(crate) mod tests { let taddr = uaddr.transparent().unwrap(); // Ensure that the wallet has at least one block - let (h, _, _) = test.generate_next_block( + let (h, _, _) = st.generate_next_block( &dfvk, AddressType::Internal, Amount::from_u64(50000).unwrap(), ); - test.scan_cached_blocks(h, 1); + st.scan_cached_blocks(h, 1); let utxo = WalletTransparentOutput::from_parts( OutPoint::new([1u8; 32], 1), @@ -1193,7 +1193,7 @@ pub(crate) mod tests { ) .unwrap(); - let res0 = test.wallet_mut().put_received_transparent_utxo(&utxo); + let res0 = st.wallet_mut().put_received_transparent_utxo(&utxo); assert!(matches!(res0, Ok(_))); let input_selector = GreedyInputSelector::new( @@ -1202,7 +1202,7 @@ pub(crate) mod tests { ); assert_matches!( - test.shield_transparent_funds( + st.shield_transparent_funds( &input_selector, NonNegativeAmount::from_u64(10000).unwrap(), &usk, diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 8bfb8c1cee..25a1be909d 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -1083,18 +1083,18 @@ mod tests { fn scan_complete() { use ScanPriority::*; - let mut test = TestBuilder::new() + let mut st = TestBuilder::new() .with_block_cache() .with_seed(Secret::new(vec![])) .with_test_account() .build(); - let dfvk = test.test_account_sapling().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); assert_matches!( // In the following, we don't care what the root hashes are, they just need to be // distinct. - test.wallet_mut().put_sapling_subtree_roots( + st.wallet_mut().put_sapling_subtree_roots( 0, &[ CommitmentTreeRoot::from_parts( @@ -1121,7 +1121,7 @@ mod tests { let initial_height = sapling_activation_height() + 310; let value = Amount::from_u64(50000).unwrap(); - test.generate_block_at( + st.generate_block_at( initial_height, BlockHash([0; 32]), &dfvk, @@ -1131,19 +1131,19 @@ mod tests { ); for _ in 1..=10 { - test.generate_next_block( + st.generate_next_block( &dfvk, AddressType::DefaultExternal, Amount::from_u64(10000).unwrap(), ); } - test.scan_cached_blocks(initial_height, 10); + st.scan_cached_blocks(initial_height, 10); // Verify the that adjacent range needed to make the note spendable has been prioritized. let sap_active = u32::from(sapling_activation_height()); assert_matches!( - test.wallet().suggest_scan_ranges(), + st.wallet().suggest_scan_ranges(), Ok(scan_ranges) if scan_ranges == vec![ scan_range((sap_active + 300)..(sap_active + 310), FoundNote) ] @@ -1151,7 +1151,7 @@ mod tests { // Check that the scanned range has been properly persisted. assert_matches!( - suggest_scan_ranges(&test.wallet().conn, Scanned), + suggest_scan_ranges(&st.wallet().conn, Scanned), Ok(scan_ranges) if scan_ranges == vec![ scan_range((sap_active + 300)..(sap_active + 310), FoundNote), scan_range((sap_active + 310)..(sap_active + 320), Scanned) @@ -1161,7 +1161,7 @@ mod tests { // Simulate the wallet going offline for a bit, update the chain tip to 20 blocks in the // future. assert_matches!( - test.wallet_mut() + st.wallet_mut() .update_chain_tip(sapling_activation_height() + 340), Ok(()) ); @@ -1169,7 +1169,7 @@ mod tests { // Check the scan range again, we should see a `ChainTip` range for the period we've been // offline. assert_matches!( - test.wallet().suggest_scan_ranges(), + st.wallet().suggest_scan_ranges(), Ok(scan_ranges) if scan_ranges == vec![ scan_range((sap_active + 320)..(sap_active + 341), ChainTip), scan_range((sap_active + 300)..(sap_active + 310), ChainTip) @@ -1178,7 +1178,7 @@ mod tests { // Now simulate a jump ahead more than 100 blocks. assert_matches!( - test.wallet_mut() + st.wallet_mut() .update_chain_tip(sapling_activation_height() + 450), Ok(()) ); @@ -1186,7 +1186,7 @@ mod tests { // Check the scan range again, we should see a `Validate` range for the previous wallet // tip, and then a `ChainTip` for the remaining range. assert_matches!( - test.wallet().suggest_scan_ranges(), + st.wallet().suggest_scan_ranges(), Ok(scan_ranges) if scan_ranges == vec![ scan_range((sap_active + 319)..(sap_active + 329), Verify), scan_range((sap_active + 329)..(sap_active + 451), ChainTip), @@ -1199,22 +1199,22 @@ mod tests { fn init_blocks_table_creates_ignored_range() { use ScanPriority::*; - let mut test = TestBuilder::new().with_seed(Secret::new(vec![])).build(); + let mut st = TestBuilder::new().with_seed(Secret::new(vec![])).build(); - let sap_active = test + let sap_active = st .wallet() .params .activation_height(NetworkUpgrade::Sapling) .unwrap(); // Initialise the blocks table. We use Canopy activation as an arbitrary birthday height // that's greater than Sapling activation. - let birthday_height = test + let birthday_height = st .wallet() .params .activation_height(NetworkUpgrade::Canopy) .unwrap(); init_blocks_table( - test.wallet_mut(), + st.wallet_mut(), birthday_height, BlockHash([1; 32]), 1, @@ -1230,12 +1230,12 @@ mod tests { ), ]; assert_matches!( - suggest_scan_ranges(&test.wallet().conn, Ignored), + suggest_scan_ranges(&st.wallet().conn, Ignored), Ok(scan_ranges) if scan_ranges == expected ); // Set up some shard history - test.wallet_mut() + st.wallet_mut() .put_sapling_subtree_roots( 0, &[ @@ -1254,12 +1254,12 @@ mod tests { .unwrap(); // Update the chain tip - let tip_height = test + let tip_height = st .wallet() .params .activation_height(NetworkUpgrade::Nu5) .unwrap(); - test.wallet_mut().update_chain_tip(tip_height).unwrap(); + st.wallet_mut().update_chain_tip(tip_height).unwrap(); // Verify that the suggested scan ranges match what is expected let expected = vec![ @@ -1290,7 +1290,7 @@ mod tests { ]; assert_matches!( - suggest_scan_ranges(&test.wallet().conn, Ignored), + suggest_scan_ranges(&st.wallet().conn, Ignored), Ok(scan_ranges) if scan_ranges == expected ); } From 8af74de93125fd6edc45f314786fb6a58594810d Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 28 Aug 2023 18:58:37 +0000 Subject: [PATCH 0113/1122] `zcash_client_sqlite`: Add `TestState::propose_shielding` Currently unused (causing an expected lint), but will be in an upcoming PR. --- zcash_client_sqlite/src/testing.rs | 34 +++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 6c4e539421..84eb383ed2 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -55,7 +55,7 @@ use zcash_primitives::{ }; #[cfg(feature = "transparent-inputs")] -use zcash_client_backend::data_api::wallet::shield_transparent_funds; +use zcash_client_backend::data_api::wallet::{propose_shielding, shield_transparent_funds}; #[cfg(feature = "transparent-inputs")] use zcash_primitives::{ legacy, legacy::keys::IncomingViewingKey, transaction::components::amount::NonNegativeAmount, @@ -422,6 +422,38 @@ impl TestState { ) } + /// Invokes [`propose_shielding`] with the given arguments. + #[cfg(feature = "transparent-inputs")] + #[allow(clippy::type_complexity)] + pub(crate) fn propose_shielding( + &mut self, + input_selector: &InputsT, + shielding_threshold: NonNegativeAmount, + from_addrs: &[TransparentAddress], + min_confirmations: NonZeroU32, + ) -> Result< + Proposal, + data_api::error::Error< + SqliteClientError, + Infallible, + InputsT::Error, + ::Error, + ReceivedNoteId, + >, + > + where + InputsT: InputSelector>, + { + propose_shielding::<_, _, _, Infallible>( + &mut self.db_data, + &self.params, + input_selector, + shielding_threshold, + from_addrs, + min_confirmations, + ) + } + /// Invokes [`create_proposed_transaction`] with the given arguments. pub(crate) fn create_proposed_transaction( &mut self, From 3be55ae96443f0d376f64510c5c788738a4421d6 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 16 Aug 2023 11:15:10 -0600 Subject: [PATCH 0114/1122] zcash_client_backend: Add test-only convenience methods for default addresses. --- zcash_client_backend/CHANGELOG.md | 20 +++++++++++--------- zcash_client_backend/src/keys.rs | 17 +++++++++++++++++ zcash_client_sqlite/Cargo.toml | 1 + 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 6980d35ef9..bb07455574 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -28,15 +28,17 @@ and this library adheres to Rust's notion of - `ScanError` - `impl ScanningKey for &K` - `impl ScanningKey for (zip32::Scope, sapling::SaplingIvk, sapling::NullifierDerivingKey)` +- Test utility functions `zcash_client_backend::keys::UnifiedSpendingKey::{default_address, + default_transparent_address}` are now available under the `test-dependencies` feature flag. ### Changed - MSRV is now 1.65.0. - Bumped dependencies to `hdwallet 0.4`, `zcash_primitives 0.12`, `zcash_note_encryption 0.4`, `incrementalmerkletree 0.4`, `orchard 0.5`, `bs58 0.5` - `zcash_client_backend::data_api`: - - `WalletRead::TxRef` has been removed in favor of consistently using `TxId` instead. + - `WalletRead::TxRef` has been removed in favor of consistently using `TxId` instead. - `WalletRead::get_transaction` now takes a `TxId` as its argument. - - `WalletWrite::{store_decrypted_tx, store_sent_tx}` now return `Result<(), Self::Error>` + - `WalletWrite::{store_decrypted_tx, store_sent_tx}` now return `Result<(), Self::Error>` as the `WalletRead::TxRef` associated type has been removed. Use `WalletRead::get_transaction` with the transaction's `TxId` instead. - `WalletRead::get_memo` now takes a `NoteId` as its argument instead of `Self::NoteRef` @@ -56,22 +58,22 @@ and this library adheres to Rust's notion of - `wallet::{create_spend_to_address, create_proposed_transaction, shield_transparent_funds}` all now require that `WalletCommitmentTrees` be implemented for the type passed to them for the `wallet_db` parameter. - - `wallet::create_proposed_transaction` now takes an additional + - `wallet::create_proposed_transaction` now takes an additional `min_confirmations` argument. - - `wallet::{spend, create_spend_to_address, shield_transparent_funds, + - `wallet::{spend, create_spend_to_address, shield_transparent_funds, propose_transfer, propose_shielding, create_proposed_transaction}` now take their respective `min_confirmations` arguments as `NonZeroU32` - `wallet::input_selection::InputSelector::{propose_transaction, propose_shielding}` now take their respective `min_confirmations` arguments as `NonZeroU32` - A new `Scan` variant has been added to `data_api::chain::error::Error`. - A new `SyncRequired` variant has been added to `data_api::wallet::input_selection::InputSelectorError`. - - The variants of the `PoolType` enum have changed; the `PoolType::Sapling` variant has been + - The variants of the `PoolType` enum have changed; the `PoolType::Sapling` variant has been removed in favor of a `PoolType::Shielded` variant that wraps a `ShieldedProtocol` value. - `zcash_client_backend::wallet`: - `SpendableNote` has been renamed to `ReceivedSaplingNote`. - Arguments to `WalletSaplingOutput::from_parts` have changed. - `zcash_client_backend::data_api::wallet::input_selection::InputSelector`: - - Arguments to `{propose_transaction, propose_shielding}` have changed. + - Arguments to `{propose_transaction, propose_shielding}` have changed. - `zcash_client_backend::data_api::wallet::{create_spend_to_address, spend, create_proposed_transaction, shield_transparent_funds}` now return the `TxId` for the newly created transaction instead an internal database identifier. @@ -85,7 +87,7 @@ and this library adheres to Rust's notion of tree and incremental witnesses for each previously-known note. In addition, the return type has now been updated to return a `Result`. - `proto/service.proto` has been updated to include the new GRPC endpoints - supported by lightwalletd v0.4.15 + supported by lightwalletd v0.4.15 ### Removed - `zcash_client_backend::data_api`: @@ -99,12 +101,12 @@ and this library adheres to Rust's notion of - `WalletWrite::advance_by_block` (use `WalletWrite::put_blocks` instead). - `PrunedBlock` has been replaced by `ScannedBlock` - `testing::MockWalletDb`, which is available under the `test-dependencies` - feature flag, has been modified by the addition of a `sapling_tree` property. + feature flag, has been modified by the addition of a `sapling_tree` property. - `wallet::input_selection`: - `Proposal::target_height` (use `Proposal::min_target_height` instead). - `zcash_client_backend::data_api::chain::validate_chain` (logic merged into `chain::scan_cached_blocks`. -- `zcash_client_backend::data_api::chain::error::{ChainError, Cause}` have been +- `zcash_client_backend::data_api::chain::error::{ChainError, Cause}` have been replaced by `zcash_client_backend::scanning::ScanError` - `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}` have been removed as individual incremental witnesses are no longer tracked on a diff --git a/zcash_client_backend/src/keys.rs b/zcash_client_backend/src/keys.rs index 285e65bb15..a850b618df 100644 --- a/zcash_client_backend/src/keys.rs +++ b/zcash_client_backend/src/keys.rs @@ -14,6 +14,9 @@ use { zcash_primitives::legacy::keys::{self as legacy, IncomingViewingKey}, }; +#[cfg(all(feature = "test-dependencies", feature = "transparent-inputs"))] +use zcash_primitives::legacy::TransparentAddress; + #[cfg(feature = "unstable")] use { byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}, @@ -336,6 +339,20 @@ impl UnifiedSpendingKey { } } } + + #[cfg(feature = "test-dependencies")] + pub fn default_address(&self) -> (UnifiedAddress, DiversifierIndex) { + self.to_unified_full_viewing_key().default_address() + } + + #[cfg(all(feature = "test-dependencies", feature = "transparent-inputs"))] + pub fn default_transparent_address(&self) -> (TransparentAddress, u32) { + self.transparent() + .to_account_pubkey() + .derive_external_ivk() + .unwrap() + .default_address() + } } /// A [ZIP 316](https://zips.z.cash/zip-0316) unified full viewing key. diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 0d409717ad..3494b19968 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -60,6 +60,7 @@ tempfile = "3.5.0" zcash_note_encryption = "0.4" zcash_proofs = { version = "0.12", path = "../zcash_proofs" } zcash_primitives = { version = "0.12", path = "../zcash_primitives", features = ["test-dependencies"] } +zcash_client_backend = { version = "0.9", path = "../zcash_client_backend", features = ["test-dependencies"] } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } [features] From a238007e14627caf2bb04c6e517f59df7680bfec Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 16 Aug 2023 11:15:10 -0600 Subject: [PATCH 0115/1122] zcash_client_sqlite: Remove `testing::network` global accessor function. --- zcash_client_sqlite/src/testing.rs | 112 ++++++++++-------- .../src/wallet/commitment_tree.rs | 8 +- zcash_client_sqlite/src/wallet/init.rs | 85 +++++++------ .../init/migrations/add_transaction_views.rs | 21 ++-- .../migrations/received_notes_nullable_nf.rs | 9 +- .../init/migrations/v_transactions_net.rs | 13 +- zcash_client_sqlite/src/wallet/sapling.rs | 14 +-- zcash_client_sqlite/src/wallet/scanning.rs | 17 +-- 8 files changed, 145 insertions(+), 134 deletions(-) diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 84eb383ed2..c1ee2ec2f8 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -1,6 +1,6 @@ use std::convert::Infallible; use std::fmt; -use std::{collections::HashMap, num::NonZeroU32}; +use std::num::NonZeroU32; #[cfg(feature = "unstable")] use std::fs::File; @@ -15,14 +15,13 @@ use tempfile::NamedTempFile; use tempfile::TempDir; #[allow(deprecated)] -use zcash_client_backend::data_api::wallet::create_spend_to_address; use zcash_client_backend::{ address::RecipientAddress, data_api::{ self, chain::{scan_cached_blocks, BlockSource}, wallet::{ - create_proposed_transaction, + create_proposed_transaction, create_spend_to_address, input_selection::{GreedyInputSelectorError, InputSelector, Proposal}, propose_transfer, spend, }, @@ -37,7 +36,7 @@ use zcash_client_backend::{ use zcash_note_encryption::Domain; use zcash_primitives::{ block::BlockHash, - consensus::{BlockHeight, Network, NetworkUpgrade, Parameters}, + consensus::{self, BlockHeight, Network, NetworkUpgrade, Parameters}, legacy::TransparentAddress, memo::MemoBytes, sapling::{ @@ -54,13 +53,6 @@ use zcash_primitives::{ zip32::{sapling::DiversifiableFullViewingKey, DiversifierIndex}, }; -#[cfg(feature = "transparent-inputs")] -use zcash_client_backend::data_api::wallet::{propose_shielding, shield_transparent_funds}; -#[cfg(feature = "transparent-inputs")] -use zcash_primitives::{ - legacy, legacy::keys::IncomingViewingKey, transaction::components::amount::NonNegativeAmount, -}; - use crate::{ chain::init::init_cache_database, error::SqliteClientError, @@ -74,13 +66,24 @@ use crate::{ use super::BlockDb; +#[cfg(feature = "transparent-inputs")] +use { + zcash_client_backend::data_api::wallet::{propose_shielding, shield_transparent_funds}, + zcash_primitives::{ + legacy::{self, keys::IncomingViewingKey}, + transaction::components::amount::NonNegativeAmount, + }, +}; + #[cfg(feature = "unstable")] use crate::{ chain::{init::init_blockmeta_db, BlockMeta}, FsBlockDb, }; + /// A builder for a `zcash_client_sqlite` test. pub(crate) struct TestBuilder { + network: Network, cache: Cache, seed: Option>, with_test_account: bool, @@ -90,6 +93,7 @@ impl TestBuilder<()> { /// Constructs a new test. pub(crate) fn new() -> Self { TestBuilder { + network: Network::TestNetwork, cache: (), seed: None, with_test_account: false, @@ -99,6 +103,7 @@ impl TestBuilder<()> { /// Adds a [`BlockDb`] cache to the test. pub(crate) fn with_block_cache(self) -> TestBuilder { TestBuilder { + network: self.network, cache: BlockCache::new(), seed: self.seed, with_test_account: self.with_test_account, @@ -109,6 +114,7 @@ impl TestBuilder<()> { #[cfg(feature = "unstable")] pub(crate) fn with_fs_block_cache(self) -> TestBuilder { TestBuilder { + network: self.network, cache: FsBlockCache::new(), seed: self.seed, with_test_account: self.with_test_account, @@ -131,10 +137,8 @@ impl TestBuilder { /// Builds the state for this test. pub(crate) fn build(self) -> TestState { - let params = network(); - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), params).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), self.network).unwrap(); init_wallet_db(&mut db_data, self.seed).unwrap(); let test_account = if self.with_test_account { @@ -145,7 +149,7 @@ impl TestBuilder { }; TestState { - params, + params: self.network, cache: self.cache, latest_cached_block: None, _data_file: data_file, @@ -186,7 +190,15 @@ where let (height, prev_hash, initial_sapling_tree_size) = self .latest_cached_block .map(|(prev_height, prev_hash, end_size)| (prev_height + 1, prev_hash, end_size)) - .unwrap_or_else(|| (sapling_activation_height(), BlockHash([0; 32]), 0)); + .unwrap_or_else(|| { + ( + self.params + .activation_height(NetworkUpgrade::Sapling) + .unwrap(), + BlockHash([0; 32]), + 0, + ) + }); let (res, nf) = self.generate_block_at( height, @@ -215,6 +227,7 @@ where initial_sapling_tree_size: u32, ) -> (Cache::InsertResult, Nullifier) { let (cb, nf) = fake_compact_block( + &self.params, height, prev_hash, dfvk, @@ -246,9 +259,18 @@ where let (height, prev_hash, initial_sapling_tree_size) = self .latest_cached_block .map(|(prev_height, prev_hash, end_size)| (prev_height + 1, prev_hash, end_size)) - .unwrap_or_else(|| (sapling_activation_height(), BlockHash([0; 32]), 0)); + .unwrap_or_else(|| { + ( + self.params + .activation_height(NetworkUpgrade::Sapling) + .unwrap(), + BlockHash([0; 32]), + 0, + ) + }); let cb = fake_compact_block_spending( + &self.params, height, prev_hash, note, @@ -308,6 +330,19 @@ impl TestState { &mut self.db_data } + /// Exposes an immutable reference to the network in use. + pub(crate) fn network(&self) -> &Network { + &self.db_data.params + } + + /// Convenience method for obtaining the Sapling activation height for the network under test. + pub(crate) fn sapling_activation_height(&self) -> BlockHeight { + self.db_data + .params + .activation_height(NetworkUpgrade::Sapling) + .expect("Sapling activation height must be known.") + } + /// Exposes the test account, if enabled via [`TestBuilder::with_test_account`]. #[cfg(feature = "unstable")] pub(crate) fn test_account( @@ -525,42 +560,20 @@ impl TestState { } } -#[cfg(feature = "mainnet")] -pub(crate) fn network() -> Network { - Network::MainNetwork -} - -#[cfg(not(feature = "mainnet"))] -pub(crate) fn network() -> Network { - Network::TestNetwork -} - -#[cfg(feature = "mainnet")] -pub(crate) fn sapling_activation_height() -> BlockHeight { - Network::MainNetwork - .activation_height(NetworkUpgrade::Sapling) - .unwrap() -} - -#[cfg(not(feature = "mainnet"))] -pub(crate) fn sapling_activation_height() -> BlockHeight { - Network::TestNetwork - .activation_height(NetworkUpgrade::Sapling) - .unwrap() -} - #[cfg(test)] pub(crate) fn init_test_accounts_table_ufvk( db_data: &mut WalletDb, ) -> (UnifiedFullViewingKey, Option) { + use std::collections::HashMap; + let seed = [0u8; 32]; let account = AccountId::from(0); - let extsk = sapling::spending_key(&seed, network().coin_type(), account); + let extsk = sapling::spending_key(&seed, db_data.params.coin_type(), account); let dfvk = extsk.to_diversifiable_full_viewing_key(); #[cfg(feature = "transparent-inputs")] let (tkey, taddr) = { - let tkey = legacy::keys::AccountPrivKey::from_seed(&network(), &seed, account) + let tkey = legacy::keys::AccountPrivKey::from_seed(&db_data.params, &seed, account) .unwrap() .to_account_pubkey(); let taddr = tkey.derive_external_ivk().unwrap().default_address().0; @@ -593,7 +606,8 @@ pub(crate) enum AddressType { /// Create a fake CompactBlock at the given height, containing a single output paying /// an address. Returns the CompactBlock and the nullifier for the new note. -pub(crate) fn fake_compact_block( +pub(crate) fn fake_compact_block( + params: &P, height: BlockHeight, prev_hash: BlockHash, dfvk: &DiversifiableFullViewingKey, @@ -609,7 +623,7 @@ pub(crate) fn fake_compact_block( // Create a fake Note for the account let mut rng = OsRng; - let rseed = generate_random_rseed(&network(), height, &mut rng); + let rseed = generate_random_rseed(params, height, &mut rng); let note = Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); let encryptor = sapling_note_encryption::<_, Network>( Some(dfvk.fvk().ovk), @@ -655,7 +669,9 @@ pub(crate) fn fake_compact_block( /// Create a fake CompactBlock at the given height, spending a single note from the /// given address. -pub(crate) fn fake_compact_block_spending( +#[allow(clippy::too_many_arguments)] +pub(crate) fn fake_compact_block_spending( + params: &P, height: BlockHeight, prev_hash: BlockHash, (nf, in_value): (Nullifier, Amount), @@ -665,7 +681,7 @@ pub(crate) fn fake_compact_block_spending( initial_sapling_tree_size: u32, ) -> CompactBlock { let mut rng = OsRng; - let rseed = generate_random_rseed(&network(), height, &mut rng); + let rseed = generate_random_rseed(params, height, &mut rng); // Create a fake CompactBlock containing the note let cspend = CompactSaplingSpend { nf: nf.to_vec() }; @@ -700,7 +716,7 @@ pub(crate) fn fake_compact_block_spending( // Create a fake Note for the change ctx.outputs.push({ let change_addr = dfvk.default_address().1; - let rseed = generate_random_rseed(&network(), height, &mut rng); + let rseed = generate_random_rseed(params, height, &mut rng); let note = Note::from_parts( change_addr, NoteValue::from_raw((in_value - value).unwrap().into()), diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 3242080719..01302dca05 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -985,14 +985,14 @@ mod tests { }; use shardtree::ShardTree; use zcash_client_backend::data_api::chain::CommitmentTreeRoot; - use zcash_primitives::consensus::BlockHeight; + use zcash_primitives::consensus::{BlockHeight, Network}; use super::SqliteShardStore; - use crate::{testing, wallet::init::init_wallet_db, WalletDb, SAPLING_TABLES_PREFIX}; + use crate::{wallet::init::init_wallet_db, WalletDb, SAPLING_TABLES_PREFIX}; fn new_tree(m: usize) -> ShardTree, 4, 3> { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); data_file.keep().unwrap(); init_wallet_db(&mut db_data, None).unwrap(); @@ -1040,7 +1040,7 @@ mod tests { #[test] fn put_shard_roots() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); data_file.keep().unwrap(); init_wallet_db(&mut db_data, None).unwrap(); diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 40913504c6..354016fe47 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -386,36 +386,27 @@ mod tests { use zcash_primitives::{ block::BlockHash, - consensus::{BlockHeight, BranchId, NetworkUpgrade, Parameters}, + consensus::{self, BlockHeight, BranchId, Network, NetworkUpgrade, Parameters}, transaction::{TransactionData, TxVersion}, zip32::sapling::ExtendedFullViewingKey, }; - use crate::{ - error::SqliteClientError, - testing::{self, network}, - wallet::scanning::priority_code, - AccountId, WalletDb, - }; + use crate::{error::SqliteClientError, wallet::scanning::priority_code, AccountId, WalletDb}; use super::{init_accounts_table, init_blocks_table, init_wallet_db}; #[cfg(feature = "transparent-inputs")] use { - crate::{ - wallet::{self, pool_code, PoolType}, - WalletWrite, - }, + crate::wallet::{self, pool_code, PoolType}, zcash_address::test_vectors, - zcash_primitives::{ - consensus::Network, legacy::keys as transparent, zip32::DiversifierIndex, - }, + zcash_client_backend::data_api::WalletWrite, + zcash_primitives::{legacy::keys as transparent, zip32::DiversifierIndex}, }; #[test] fn verify_schema() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); use regex::Regex; @@ -609,7 +600,7 @@ mod tests { AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height ) WHERE scan_queue.priority > {}", - u32::from(testing::network().activation_height(NetworkUpgrade::Sapling).unwrap()), + u32::from(db_data.params.activation_height(NetworkUpgrade::Sapling).unwrap()), priority_code(&ScanPriority::Scanned), ), // v_transactions @@ -768,7 +759,7 @@ mod tests { #[test] fn init_migrate_from_0_3_0() { - fn init_0_3_0

( + fn init_0_3_0( wdb: &mut WalletDb, extfvk: &ExtendedFullViewingKey, account: AccountId, @@ -852,11 +843,11 @@ mod tests { )?; let address = encode_payment_address( - testing::network().hrp_sapling_payment_address(), + wdb.params.hrp_sapling_payment_address(), &extfvk.default_address().1, ); let extfvk = encode_extended_full_viewing_key( - testing::network().hrp_sapling_extended_full_viewing_key(), + wdb.params.hrp_sapling_extended_full_viewing_key(), extfvk, ); wdb.conn.execute( @@ -872,19 +863,21 @@ mod tests { Ok(()) } + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); + let seed = [0xab; 32]; let account = AccountId::from(0); - let secret_key = sapling::spending_key(&seed, testing::network().coin_type(), account); + let secret_key = sapling::spending_key(&seed, db_data.params.coin_type(), account); let extfvk = secret_key.to_extended_full_viewing_key(); - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); + init_0_3_0(&mut db_data, &extfvk, account).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(seed.to_vec()))).unwrap(); } #[test] fn init_migrate_from_autoshielding_poc() { - fn init_autoshielding

( + fn init_autoshielding( wdb: &mut WalletDb, extfvk: &ExtendedFullViewingKey, account: AccountId, @@ -984,11 +977,11 @@ mod tests { )?; let address = encode_payment_address( - testing::network().hrp_sapling_payment_address(), + wdb.params.hrp_sapling_payment_address(), &extfvk.default_address().1, ); let extfvk = encode_extended_full_viewing_key( - testing::network().hrp_sapling_extended_full_viewing_key(), + wdb.params.hrp_sapling_extended_full_viewing_key(), extfvk, ); wdb.conn.execute( @@ -1038,19 +1031,21 @@ mod tests { Ok(()) } + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); + let seed = [0xab; 32]; let account = AccountId::from(0); - let secret_key = sapling::spending_key(&seed, testing::network().coin_type(), account); + let secret_key = sapling::spending_key(&seed, db_data.params.coin_type(), account); let extfvk = secret_key.to_extended_full_viewing_key(); - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); + init_autoshielding(&mut db_data, &extfvk, account).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(seed.to_vec()))).unwrap(); } #[test] fn init_migrate_from_main_pre_migrations() { - fn init_main

( + fn init_main( wdb: &mut WalletDb, ufvk: &UnifiedFullViewingKey, account: AccountId, @@ -1150,9 +1145,9 @@ mod tests { [], )?; - let ufvk_str = ufvk.encode(&testing::network()); + let ufvk_str = ufvk.encode(&wdb.params); let address_str = - RecipientAddress::Unified(ufvk.default_address().0).encode(&testing::network()); + RecipientAddress::Unified(ufvk.default_address().0).encode(&wdb.params); wdb.conn.execute( "INSERT INTO accounts (account, ufvk, address, transparent_address) VALUES (?, ?, ?, '')", @@ -1168,7 +1163,7 @@ mod tests { { let taddr = RecipientAddress::Transparent(*ufvk.default_address().0.transparent().unwrap()) - .encode(&testing::network()); + .encode(&wdb.params); wdb.conn.execute( "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'000000')", [], @@ -1186,12 +1181,13 @@ mod tests { Ok(()) } + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); + let seed = [0xab; 32]; let account = AccountId::from(0); - let secret_key = - UnifiedSpendingKey::from_seed(&testing::network(), &seed, account).unwrap(); - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); + let secret_key = UnifiedSpendingKey::from_seed(&db_data.params, &seed, account).unwrap(); + init_main( &mut db_data, &secret_key.to_unified_full_viewing_key(), @@ -1204,7 +1200,7 @@ mod tests { #[test] fn init_accounts_table_only_works_once() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // We can call the function as many times as we want with no data @@ -1215,13 +1211,13 @@ mod tests { let account = AccountId::from(0); // First call with data should initialise the accounts table - let extsk = sapling::spending_key(&seed, network().coin_type(), account); + let extsk = sapling::spending_key(&seed, db_data.params.coin_type(), account); let dfvk = extsk.to_diversifiable_full_viewing_key(); #[cfg(feature = "transparent-inputs")] let ufvk = UnifiedFullViewingKey::new( Some( - transparent::AccountPrivKey::from_seed(&network(), &seed, account) + transparent::AccountPrivKey::from_seed(&db_data.params, &seed, account) .unwrap() .to_account_pubkey(), ), @@ -1243,8 +1239,9 @@ mod tests { #[test] fn init_accounts_table_allows_no_gaps() { + let params = Network::TestNetwork; let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), params).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // allow sequential initialization @@ -1253,7 +1250,7 @@ mod tests { ids.iter() .map(|a| { let account = AccountId::from(*a); - UnifiedSpendingKey::from_seed(&network(), &seed, account) + UnifiedSpendingKey::from_seed(¶ms, &seed, account) .map(|k| (account, k.to_unified_full_viewing_key())) .unwrap() }) @@ -1273,7 +1270,7 @@ mod tests { #[test] fn init_blocks_table_only_works_once() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // First call with data should initialise the blocks table @@ -1300,14 +1297,14 @@ mod tests { #[test] fn init_accounts_table_stores_correct_address() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); init_wallet_db(&mut db_data, None).unwrap(); let seed = [0u8; 32]; // Add an account to the wallet let account_id = AccountId::from(0); - let usk = UnifiedSpendingKey::from_seed(&testing::network(), &seed, account_id).unwrap(); + let usk = UnifiedSpendingKey::from_seed(&db_data.params, &seed, account_id).unwrap(); let ufvk = usk.to_unified_full_viewing_key(); let expected_address = ufvk.sapling().unwrap().default_address().1; let ufvks = HashMap::from([(account_id, ufvk)]); diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs index 0d217c545a..05ab54aed5 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs @@ -283,10 +283,9 @@ mod tests { use tempfile::NamedTempFile; use zcash_client_backend::keys::UnifiedSpendingKey; - use zcash_primitives::zip32::AccountId; + use zcash_primitives::{consensus::Network, zip32::AccountId}; use crate::{ - testing, wallet::init::{init_wallet_db_internal, migrations::addresses_table}, WalletDb, }; @@ -310,19 +309,19 @@ mod tests { #[test] fn transaction_views() { + let network = Network::TestNetwork; let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), network).unwrap(); init_wallet_db_internal(&mut db_data, None, &[addresses_table::MIGRATION_ID]).unwrap(); let usk = - UnifiedSpendingKey::from_seed(&testing::network(), &[0u8; 32][..], AccountId::from(0)) - .unwrap(); + UnifiedSpendingKey::from_seed(&network, &[0u8; 32][..], AccountId::from(0)).unwrap(); let ufvk = usk.to_unified_full_viewing_key(); db_data .conn .execute( "INSERT INTO accounts (account, ufvk) VALUES (0, ?)", - params![ufvk.encode(&testing::network())], + params![ufvk.encode(&network)], ) .unwrap(); @@ -402,8 +401,9 @@ mod tests { #[test] #[cfg(feature = "transparent-inputs")] fn migrate_from_wm2() { + let network = Network::TestNetwork; let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), network).unwrap(); init_wallet_db_internal( &mut db_data, None, @@ -440,8 +440,7 @@ mod tests { tx.write(&mut tx_bytes).unwrap(); let usk = - UnifiedSpendingKey::from_seed(&testing::network(), &[0u8; 32][..], AccountId::from(0)) - .unwrap(); + UnifiedSpendingKey::from_seed(&network, &[0u8; 32][..], AccountId::from(0)).unwrap(); let ufvk = usk.to_unified_full_viewing_key(); let (ua, _) = ufvk.default_address(); let taddr = ufvk @@ -451,11 +450,11 @@ mod tests { .ok() .map(|k| k.derive_address(0).unwrap()) }) - .map(|a| a.encode(&testing::network())); + .map(|a| a.encode(&network)); db_data.conn.execute( "INSERT INTO accounts (account, ufvk, address, transparent_address) VALUES (0, ?, ?, ?)", - params![ufvk.encode(&testing::network()), ua.encode(&testing::network()), &taddr] + params![ufvk.encode(&network), ua.encode(&network), &taddr] ).unwrap(); db_data .conn diff --git a/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs b/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs index 2c946f3aa9..5a856dc3d2 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs @@ -233,10 +233,9 @@ mod tests { use tempfile::NamedTempFile; use zcash_client_backend::keys::UnifiedSpendingKey; - use zcash_primitives::zip32::AccountId; + use zcash_primitives::{consensus::Network, zip32::AccountId}; use crate::{ - testing, wallet::init::{init_wallet_db_internal, migrations::v_transactions_net}, WalletDb, }; @@ -244,19 +243,19 @@ mod tests { #[test] fn received_notes_nullable_migration() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); init_wallet_db_internal(&mut db_data, None, &[v_transactions_net::MIGRATION_ID]).unwrap(); // Create an account in the wallet let usk0 = - UnifiedSpendingKey::from_seed(&testing::network(), &[0u8; 32][..], AccountId::from(0)) + UnifiedSpendingKey::from_seed(&db_data.params, &[0u8; 32][..], AccountId::from(0)) .unwrap(); let ufvk0 = usk0.to_unified_full_viewing_key(); db_data .conn .execute( "INSERT INTO accounts (account, ufvk) VALUES (0, ?)", - params![ufvk0.encode(&testing::network())], + params![ufvk0.encode(&db_data.params)], ) .unwrap(); diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs index b6bfe4a197..14cd830b13 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs @@ -212,10 +212,9 @@ mod tests { use tempfile::NamedTempFile; use zcash_client_backend::keys::UnifiedSpendingKey; - use zcash_primitives::zip32::AccountId; + use zcash_primitives::{consensus::Network, zip32::AccountId}; use crate::{ - testing, wallet::init::{init_wallet_db_internal, migrations::add_transaction_views}, WalletDb, }; @@ -223,32 +222,32 @@ mod tests { #[test] fn v_transactions_net() { let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), testing::network()).unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); init_wallet_db_internal(&mut db_data, None, &[add_transaction_views::MIGRATION_ID]) .unwrap(); // Create two accounts in the wallet. let usk0 = - UnifiedSpendingKey::from_seed(&testing::network(), &[0u8; 32][..], AccountId::from(0)) + UnifiedSpendingKey::from_seed(&db_data.params, &[0u8; 32][..], AccountId::from(0)) .unwrap(); let ufvk0 = usk0.to_unified_full_viewing_key(); db_data .conn .execute( "INSERT INTO accounts (account, ufvk) VALUES (0, ?)", - params![ufvk0.encode(&testing::network())], + params![ufvk0.encode(&db_data.params)], ) .unwrap(); let usk1 = - UnifiedSpendingKey::from_seed(&testing::network(), &[1u8; 32][..], AccountId::from(1)) + UnifiedSpendingKey::from_seed(&db_data.params, &[1u8; 32][..], AccountId::from(1)) .unwrap(); let ufvk1 = usk1.to_unified_full_viewing_key(); db_data .conn .execute( "INSERT INTO accounts (account, ufvk) VALUES (1, ?)", - params![ufvk1.encode(&testing::network())], + params![ufvk1.encode(&db_data.params)], ) .unwrap(); diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index edf4ac2056..8dfd296458 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -439,7 +439,7 @@ pub(crate) mod tests { use crate::{ error::SqliteClientError, - testing::{self, network, AddressType, BlockCache, TestBuilder, TestState}, + testing::{AddressType, BlockCache, TestBuilder, TestState}, wallet::{commitment_tree, get_balance, get_balance_at}, AccountId, NoteId, ReceivedNoteId, }; @@ -532,7 +532,7 @@ pub(crate) mod tests { let ufvks = [(account, usk.to_unified_full_viewing_key())] .into_iter() .collect(); - let decrypted_outputs = decrypt_transaction(&testing::network(), h + 1, &tx, &ufvks); + let decrypted_outputs = decrypt_transaction(st.network(), h + 1, &tx, &ufvks); assert_eq!(decrypted_outputs.len(), 2); let mut found_tx_change_memo = false; @@ -619,7 +619,7 @@ pub(crate) mod tests { // Create a USK that doesn't exist in the wallet let acct1 = AccountId::from(1); - let usk1 = UnifiedSpendingKey::from_seed(&network(), &[1u8; 32], acct1).unwrap(); + let usk1 = UnifiedSpendingKey::from_seed(st.network(), &[1u8; 32], acct1).unwrap(); // Attempting to spend with a USK that is not in the wallet results in an error assert_matches!( @@ -900,7 +900,7 @@ pub(crate) mod tests { let to = addr2.into(); #[allow(clippy::type_complexity)] - let send_and_recover_with_policy = |test: &mut TestState, + let send_and_recover_with_policy = |st: &mut TestState, ovk_policy| -> Result< Option<(Note, PaymentAddress, MemoBytes)>, @@ -912,7 +912,7 @@ pub(crate) mod tests { ReceivedNoteId, >, > { - let txid = test.create_spend_to_address( + let txid = st.create_spend_to_address( &usk, &to, Amount::from_u64(15000).unwrap(), @@ -922,7 +922,7 @@ pub(crate) mod tests { )?; // Fetch the transaction from the database - let raw_tx: Vec<_> = test + let raw_tx: Vec<_> = st .wallet() .conn .query_row( @@ -937,7 +937,7 @@ pub(crate) mod tests { for output in tx.sapling_bundle().unwrap().shielded_outputs() { // Find the output that decrypts with the external OVK let result = try_sapling_output_recovery( - &network(), + st.network(), h1, &dfvk.to_ovk(Scope::External), output, diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 25a1be909d..48ad6d4ae5 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -752,7 +752,7 @@ mod tests { }; use crate::{ - testing::{sapling_activation_height, AddressType, TestBuilder}, + testing::{AddressType, TestBuilder}, wallet::{init::init_blocks_table, scanning::suggest_scan_ranges}, }; @@ -1090,6 +1090,7 @@ mod tests { .build(); let dfvk = st.test_account_sapling().unwrap(); + let sapling_activation_height = st.sapling_activation_height(); assert_matches!( // In the following, we don't care what the root hashes are, they just need to be @@ -1098,15 +1099,15 @@ mod tests { 0, &[ CommitmentTreeRoot::from_parts( - sapling_activation_height() + 100, + sapling_activation_height + 100, Node::empty_root(Level::from(0)) ), CommitmentTreeRoot::from_parts( - sapling_activation_height() + 200, + sapling_activation_height + 200, Node::empty_root(Level::from(1)) ), CommitmentTreeRoot::from_parts( - sapling_activation_height() + 300, + sapling_activation_height + 300, Node::empty_root(Level::from(2)) ), ] @@ -1118,7 +1119,7 @@ mod tests { // of 10 blocks. After `scan_cached_blocks`, the scan queue should have a requested scan // range of 300..310 with `FoundNote` priority, 310..320 with `Scanned` priority. let initial_sapling_tree_size = (0x1 << 16) * 3 + 5; - let initial_height = sapling_activation_height() + 310; + let initial_height = sapling_activation_height + 310; let value = Amount::from_u64(50000).unwrap(); st.generate_block_at( @@ -1141,7 +1142,7 @@ mod tests { st.scan_cached_blocks(initial_height, 10); // Verify the that adjacent range needed to make the note spendable has been prioritized. - let sap_active = u32::from(sapling_activation_height()); + let sap_active = u32::from(sapling_activation_height); assert_matches!( st.wallet().suggest_scan_ranges(), Ok(scan_ranges) if scan_ranges == vec![ @@ -1162,7 +1163,7 @@ mod tests { // future. assert_matches!( st.wallet_mut() - .update_chain_tip(sapling_activation_height() + 340), + .update_chain_tip(sapling_activation_height + 340), Ok(()) ); @@ -1179,7 +1180,7 @@ mod tests { // Now simulate a jump ahead more than 100 blocks. assert_matches!( st.wallet_mut() - .update_chain_tip(sapling_activation_height() + 450), + .update_chain_tip(sapling_activation_height + 450), Ok(()) ); From ff8104fa7569a1245919de2172410684329f5495 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 16 Aug 2023 11:15:10 -0600 Subject: [PATCH 0116/1122] zcash_client_sqlite: (testing) Initialize wallets using an `AccountBirthday` struct. --- zcash_client_backend/CHANGELOG.md | 1 + zcash_client_backend/src/data_api.rs | 30 +++++- zcash_client_sqlite/src/chain.rs | 25 ++--- zcash_client_sqlite/src/lib.rs | 21 +++-- zcash_client_sqlite/src/testing.rs | 103 ++++++--------------- zcash_client_sqlite/src/wallet.rs | 8 +- zcash_client_sqlite/src/wallet/sapling.rs | 7 +- zcash_client_sqlite/src/wallet/scanning.rs | 36 +++---- 8 files changed, 104 insertions(+), 127 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index bb07455574..8dfa7c49ba 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -11,6 +11,7 @@ and this library adheres to Rust's notion of - `impl Eq for zcash_client_backend::zip321::{Payment, TransactionRequest}` - `impl Debug` for `zcash_client_backend::{data_api::wallet::input_selection::Proposal, wallet::ReceivedSaplingNote}` - `zcash_client_backend::data_api`: + - `AccountBirthday` - `BlockMetadata` - `NoteId` - `NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index d1711375f7..10b4285558 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use std::fmt::Debug; use std::num::NonZeroU32; -use incrementalmerkletree::Retention; +use incrementalmerkletree::{frontier::Frontier, Retention}; use secrecy::SecretVec; use shardtree::{error::ShardTreeError, store::ShardStore, ShardTree}; use zcash_primitives::{ @@ -12,7 +12,7 @@ use zcash_primitives::{ consensus::BlockHeight, legacy::TransparentAddress, memo::{Memo, MemoBytes}, - sapling, + sapling::{self, Node, NOTE_COMMITMENT_TREE_DEPTH}, transaction::{ components::{amount::Amount, OutPoint}, Transaction, TxId, @@ -466,6 +466,32 @@ impl SentTransactionOutput { } } +#[derive(Clone, Debug)] +pub struct AccountBirthday { + height: BlockHeight, + sapling_frontier: Frontier, +} + +impl AccountBirthday { + pub fn from_parts( + height: BlockHeight, + sapling_frontier: Frontier, + ) -> Self { + Self { + height, + sapling_frontier, + } + } + + pub fn sapling_frontier(&self) -> &Frontier { + &self.sapling_frontier + } + + pub fn height(&self) -> BlockHeight { + self.height + } +} + /// This trait encapsulates the write capabilities required to update stored /// wallet data. pub trait WalletWrite: WalletRead { diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 519f891bb2..c2b79467fd 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -345,7 +345,7 @@ mod tests { }; use crate::{ - testing::{AddressType, TestBuilder}, + testing::{birthday_at_sapling_activation, AddressType, TestBuilder}, wallet::{get_balance, truncate_to_height}, AccountId, }; @@ -354,8 +354,7 @@ mod tests { fn valid_chain_states() { let mut st = TestBuilder::new() .with_block_cache() - .with_seed(Secret::new(vec![])) - .with_test_account() + .with_test_account(birthday_at_sapling_activation) .build(); let dfvk = st.test_account_sapling().unwrap(); @@ -388,8 +387,7 @@ mod tests { fn invalid_chain_cache_disconnected() { let mut st = TestBuilder::new() .with_block_cache() - .with_seed(Secret::new(vec![])) - .with_test_account() + .with_test_account(birthday_at_sapling_activation) .build(); let dfvk = st.test_account_sapling().unwrap(); @@ -440,8 +438,7 @@ mod tests { fn data_db_truncation() { let mut st = TestBuilder::new() .with_block_cache() - .with_seed(Secret::new(vec![])) - .with_test_account() + .with_test_account(birthday_at_sapling_activation) .build(); let dfvk = st.test_account_sapling().unwrap(); @@ -501,10 +498,7 @@ mod tests { #[test] fn scan_cached_blocks_allows_blocks_out_of_order() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_seed(Secret::new(vec![])) - .build(); + let mut st = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); @@ -564,8 +558,7 @@ mod tests { fn scan_cached_blocks_finds_received_notes() { let mut st = TestBuilder::new() .with_block_cache() - .with_seed(Secret::new(vec![])) - .with_test_account() + .with_test_account(birthday_at_sapling_activation) .build(); let dfvk = st.test_account_sapling().unwrap(); @@ -607,8 +600,7 @@ mod tests { fn scan_cached_blocks_finds_change_notes() { let mut st = TestBuilder::new() .with_block_cache() - .with_seed(Secret::new(vec![])) - .with_test_account() + .with_test_account(birthday_at_sapling_activation) .build(); let dfvk = st.test_account_sapling().unwrap(); @@ -653,8 +645,7 @@ mod tests { fn scan_cached_blocks_detects_spends_out_of_order() { let mut st = TestBuilder::new() .with_block_cache() - .with_seed(Secret::new(vec![])) - .with_test_account() + .with_test_account(birthday_at_sapling_activation) .build(); let dfvk = st.test_account_sapling().unwrap(); diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index dde55aa635..93045c4c80 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -1087,7 +1087,10 @@ extern crate assert_matches; mod tests { use zcash_client_backend::data_api::{WalletRead, WalletWrite}; - use crate::{testing::TestBuilder, AccountId}; + use crate::{ + testing::{birthday_at_sapling_activation, TestBuilder}, + AccountId, + }; #[cfg(feature = "unstable")] use zcash_primitives::{consensus::Parameters, transaction::components::Amount}; @@ -1100,7 +1103,9 @@ mod tests { #[test] pub(crate) fn get_next_available_address() { - let mut st = TestBuilder::new().with_test_account().build(); + let mut st = TestBuilder::new() + .with_test_account(birthday_at_sapling_activation) + .build(); let account = AccountId::from(0); let current_addr = st.wallet().get_current_address(account).unwrap(); @@ -1117,17 +1122,15 @@ mod tests { #[cfg(feature = "transparent-inputs")] #[test] fn transparent_receivers() { - use secrecy::Secret; - + // Add an account to the wallet. let st = TestBuilder::new() .with_block_cache() - .with_seed(Secret::new(vec![])) - .with_test_account() + .with_test_account(birthday_at_sapling_activation) .build(); - // Add an account to the wallet. - let (ufvk, taddr) = st.test_account().unwrap(); - let taddr = taddr.unwrap(); + let (_, usk, _) = st.test_account().unwrap(); + let ufvk = usk.to_unified_full_viewing_key(); + let (taddr, _) = usk.default_transparent_address(); let receivers = st.wallet().get_transparent_receivers(0.into()).unwrap(); diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index c1ee2ec2f8..e74bcd9764 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -8,7 +8,7 @@ use std::fs::File; use prost::Message; use rand_core::{OsRng, RngCore}; use rusqlite::{params, Connection}; -use secrecy::SecretVec; +use secrecy::Secret; use tempfile::NamedTempFile; #[cfg(feature = "unstable")] @@ -25,8 +25,9 @@ use zcash_client_backend::{ input_selection::{GreedyInputSelectorError, InputSelector, Proposal}, propose_transfer, spend, }, + AccountBirthday, WalletWrite, }, - keys::{sapling, UnifiedFullViewingKey, UnifiedSpendingKey}, + keys::UnifiedSpendingKey, proto::compact_formats::{ self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }, @@ -56,11 +57,7 @@ use zcash_primitives::{ use crate::{ chain::init::init_cache_database, error::SqliteClientError, - wallet::{ - commitment_tree, - init::{init_accounts_table, init_wallet_db}, - sapling::tests::test_prover, - }, + wallet::{commitment_tree, init::init_wallet_db, sapling::tests::test_prover}, AccountId, ReceivedNoteId, WalletDb, }; @@ -69,10 +66,7 @@ use super::BlockDb; #[cfg(feature = "transparent-inputs")] use { zcash_client_backend::data_api::wallet::{propose_shielding, shield_transparent_funds}, - zcash_primitives::{ - legacy::{self, keys::IncomingViewingKey}, - transaction::components::amount::NonNegativeAmount, - }, + zcash_primitives::transaction::components::amount::NonNegativeAmount, }; #[cfg(feature = "unstable")] @@ -85,8 +79,7 @@ use crate::{ pub(crate) struct TestBuilder { network: Network, cache: Cache, - seed: Option>, - with_test_account: bool, + test_account_birthday: Option, } impl TestBuilder<()> { @@ -95,8 +88,7 @@ impl TestBuilder<()> { TestBuilder { network: Network::TestNetwork, cache: (), - seed: None, - with_test_account: false, + test_account_birthday: None, } } @@ -105,8 +97,7 @@ impl TestBuilder<()> { TestBuilder { network: self.network, cache: BlockCache::new(), - seed: self.seed, - with_test_account: self.with_test_account, + test_account_birthday: self.test_account_birthday, } } @@ -116,22 +107,17 @@ impl TestBuilder<()> { TestBuilder { network: self.network, cache: FsBlockCache::new(), - seed: self.seed, - with_test_account: self.with_test_account, + test_account_birthday: self.test_account_birthday, } } } impl TestBuilder { - /// Gives the test knowledge of the wallet seed for initialization. - pub(crate) fn with_seed(mut self, seed: SecretVec) -> Self { - // TODO remove - self.seed = Some(seed); - self - } - - pub(crate) fn with_test_account(mut self) -> Self { - self.with_test_account = true; + pub(crate) fn with_test_account AccountBirthday>( + mut self, + birthday: F, + ) -> Self { + self.test_account_birthday = Some(birthday(&self.network)); self } @@ -139,11 +125,12 @@ impl TestBuilder { pub(crate) fn build(self) -> TestState { let data_file = NamedTempFile::new().unwrap(); let mut db_data = WalletDb::for_path(data_file.path(), self.network).unwrap(); - init_wallet_db(&mut db_data, self.seed).unwrap(); + init_wallet_db(&mut db_data, None).unwrap(); - let test_account = if self.with_test_account { - // Add an account to the wallet - Some(init_test_accounts_table_ufvk(&mut db_data)) + let test_account = if let Some(birthday) = self.test_account_birthday { + let seed = Secret::new(vec![0u8; 32]); + let (account, usk) = db_data.create_account(&seed).unwrap(); + Some((account, usk, birthday)) } else { None }; @@ -166,7 +153,7 @@ pub(crate) struct TestState { latest_cached_block: Option<(BlockHeight, BlockHash, u32)>, _data_file: NamedTempFile, db_data: WalletDb, - test_account: Option<(UnifiedFullViewingKey, Option)>, + test_account: Option<(AccountId, UnifiedSpendingKey, AccountBirthday)>, } impl TestState @@ -293,8 +280,7 @@ where /// Invokes [`scan_cached_blocks`] with the given arguments, expecting success. pub(crate) fn scan_cached_blocks(&mut self, from_height: BlockHeight, limit: usize) { - self.try_scan_cached_blocks(from_height, limit) - .expect("should succeed for this test"); + assert_matches!(self.try_scan_cached_blocks(from_height, limit), Ok(_)); } /// Invokes [`scan_cached_blocks`] with the given arguments. @@ -344,10 +330,7 @@ impl TestState { } /// Exposes the test account, if enabled via [`TestBuilder::with_test_account`]. - #[cfg(feature = "unstable")] - pub(crate) fn test_account( - &self, - ) -> Option<(UnifiedFullViewingKey, Option)> { + pub(crate) fn test_account(&self) -> Option<(AccountId, UnifiedSpendingKey, AccountBirthday)> { self.test_account.as_ref().cloned() } @@ -355,7 +338,7 @@ impl TestState { pub(crate) fn test_account_sapling(&self) -> Option { self.test_account .as_ref() - .map(|(ufvk, _)| ufvk.sapling().unwrap().clone()) + .and_then(|(_, usk, _)| usk.to_unified_full_viewing_key().sapling().cloned()) } /// Invokes [`create_spend_to_address`] with the given arguments. @@ -561,40 +544,14 @@ impl TestState { } #[cfg(test)] -pub(crate) fn init_test_accounts_table_ufvk( - db_data: &mut WalletDb, -) -> (UnifiedFullViewingKey, Option) { - use std::collections::HashMap; - - let seed = [0u8; 32]; - let account = AccountId::from(0); - let extsk = sapling::spending_key(&seed, db_data.params.coin_type(), account); - let dfvk = extsk.to_diversifiable_full_viewing_key(); - - #[cfg(feature = "transparent-inputs")] - let (tkey, taddr) = { - let tkey = legacy::keys::AccountPrivKey::from_seed(&db_data.params, &seed, account) - .unwrap() - .to_account_pubkey(); - let taddr = tkey.derive_external_ivk().unwrap().default_address().0; - (Some(tkey), Some(taddr)) - }; - - #[cfg(not(feature = "transparent-inputs"))] - let taddr = None; - - let ufvk = UnifiedFullViewingKey::new( - #[cfg(feature = "transparent-inputs")] - tkey, - Some(dfvk), - None, +pub(crate) fn birthday_at_sapling_activation( + params: &P, +) -> AccountBirthday { + use incrementalmerkletree::frontier::Frontier; + AccountBirthday::from_parts( + params.activation_height(NetworkUpgrade::Sapling).unwrap(), + Frontier::empty(), ) - .unwrap(); - - let ufvks = HashMap::from([(account, ufvk.clone())]); - init_accounts_table(db_data, &ufvks).unwrap(); - - (ufvk, taddr) } #[allow(dead_code)] diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 0434c2c6b6..e1f8e19c36 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1577,7 +1577,10 @@ mod tests { use zcash_client_backend::data_api::WalletRead; - use crate::{testing::TestBuilder, AccountId}; + use crate::{ + testing::{birthday_at_sapling_activation, TestBuilder}, + AccountId, + }; use super::get_balance; @@ -1595,8 +1598,7 @@ mod tests { #[test] fn empty_database_has_no_balance() { let st = TestBuilder::new() - .with_seed(Secret::new(vec![])) - .with_test_account() + .with_test_account(birthday_at_sapling_activation) .build(); // The account should be empty diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 8dfd296458..db1d126c74 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -609,7 +609,7 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_on_incorrect_usk() { - let mut st = TestBuilder::new().with_seed(Secret::new(vec![])).build(); + let mut st = TestBuilder::new().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); @@ -778,10 +778,7 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_on_locked_notes() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_seed(Secret::new(vec![])) - .build(); + let mut st = TestBuilder::new().with_block_cache().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 48ad6d4ae5..ff1d91c851 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -739,6 +739,7 @@ mod tests { use incrementalmerkletree::{Hashable, Level}; use secrecy::Secret; + use zcash_client_backend::data_api::{ chain::CommitmentTreeRoot, scanning::{ScanPriority, ScanRange}, @@ -752,7 +753,7 @@ mod tests { }; use crate::{ - testing::{AddressType, TestBuilder}, + testing::{birthday_at_sapling_activation, AddressType, TestBuilder}, wallet::{init::init_blocks_table, scanning::suggest_scan_ranges}, }; @@ -1085,8 +1086,7 @@ mod tests { let mut st = TestBuilder::new() .with_block_cache() - .with_seed(Secret::new(vec![])) - .with_test_account() + .with_test_account(birthday_at_sapling_activation) .build(); let dfvk = st.test_account_sapling().unwrap(); @@ -1197,23 +1197,21 @@ mod tests { } #[test] - fn init_blocks_table_creates_ignored_range() { + fn create_account_creates_ignored_range() { use ScanPriority::*; - let mut st = TestBuilder::new().with_seed(Secret::new(vec![])).build(); + let mut st = TestBuilder::new().with_block_cache().build(); - let sap_active = st - .wallet() - .params - .activation_height(NetworkUpgrade::Sapling) - .unwrap(); - // Initialise the blocks table. We use Canopy activation as an arbitrary birthday height - // that's greater than Sapling activation. + let sap_active = st.sapling_activation_height(); + + // We use Canopy activation as an arbitrary birthday height that's greater than Sapling + // activation. let birthday_height = st - .wallet() - .params + .network() .activation_height(NetworkUpgrade::Canopy) .unwrap(); + + // call `init_blocks_table` to initialize the scan queue init_blocks_table( st.wallet_mut(), birthday_height, @@ -1223,6 +1221,10 @@ mod tests { ) .unwrap(); + let seed = Secret::new(vec![0u8; 32]); + let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); + let _dfvk = usk.to_unified_full_viewing_key().sapling().unwrap().clone(); + let expected = vec![ // The range up to and including the wallet's birthday height is ignored. scan_range( @@ -1290,9 +1292,7 @@ mod tests { ), ]; - assert_matches!( - suggest_scan_ranges(&st.wallet().conn, Ignored), - Ok(scan_ranges) if scan_ranges == expected - ); + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, expected); } } From 22b6ddd7544f9a8da8d5069b28feb7901f7d6fde Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 29 Aug 2023 16:32:18 -0600 Subject: [PATCH 0117/1122] Apply suggestions from code review Co-authored-by: str4d --- zcash_client_backend/src/data_api.rs | 11 +++++ zcash_client_sqlite/src/testing.rs | 52 +++++++++------------- zcash_client_sqlite/src/wallet/init.rs | 17 ++++--- zcash_client_sqlite/src/wallet/sapling.rs | 6 +-- zcash_client_sqlite/src/wallet/scanning.rs | 2 +- 5 files changed, 45 insertions(+), 43 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 10b4285558..cda26813b9 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -466,6 +466,8 @@ impl SentTransactionOutput { } } +/// A data structure used to set the birthday height for an account, and ensure that the initial +/// note commitment tree state is recorded at that height. #[derive(Clone, Debug)] pub struct AccountBirthday { height: BlockHeight, @@ -473,6 +475,12 @@ pub struct AccountBirthday { } impl AccountBirthday { + /// Constructs a new [`AccountBirthday`] from its constituent parts. + /// + /// * `height`: The birthday height of the account. This is defined as the height of the last + /// block that is known to contain no transactions sent to addresses belonging to the account. + /// * `sapling_frontier`: The Sapling note commitment tree frontier as of the end of the block + /// at `height`. pub fn from_parts( height: BlockHeight, sapling_frontier: Frontier, @@ -483,10 +491,13 @@ impl AccountBirthday { } } + /// Returns the Sapling note commitment tree frontier as of the end of the block at + /// [`Self::height`]. pub fn sapling_frontier(&self) -> &Frontier { &self.sapling_frontier } + /// Returns the birthday height of the account. pub fn height(&self) -> BlockHeight { self.height } diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index e74bcd9764..b98e546ec2 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -136,7 +136,6 @@ impl TestBuilder { }; TestState { - params: self.network, cache: self.cache, latest_cached_block: None, _data_file: data_file, @@ -148,7 +147,6 @@ impl TestBuilder { /// The state for a `zcash_client_sqlite` test. pub(crate) struct TestState { - params: Network, cache: Cache, latest_cached_block: Option<(BlockHeight, BlockHash, u32)>, _data_file: NamedTempFile, @@ -177,15 +175,7 @@ where let (height, prev_hash, initial_sapling_tree_size) = self .latest_cached_block .map(|(prev_height, prev_hash, end_size)| (prev_height + 1, prev_hash, end_size)) - .unwrap_or_else(|| { - ( - self.params - .activation_height(NetworkUpgrade::Sapling) - .unwrap(), - BlockHash([0; 32]), - 0, - ) - }); + .unwrap_or_else(|| (self.sapling_activation_height(), BlockHash([0; 32]), 0)); let (res, nf) = self.generate_block_at( height, @@ -214,7 +204,7 @@ where initial_sapling_tree_size: u32, ) -> (Cache::InsertResult, Nullifier) { let (cb, nf) = fake_compact_block( - &self.params, + &self.network(), height, prev_hash, dfvk, @@ -246,18 +236,10 @@ where let (height, prev_hash, initial_sapling_tree_size) = self .latest_cached_block .map(|(prev_height, prev_hash, end_size)| (prev_height + 1, prev_hash, end_size)) - .unwrap_or_else(|| { - ( - self.params - .activation_height(NetworkUpgrade::Sapling) - .unwrap(), - BlockHash([0; 32]), - 0, - ) - }); + .unwrap_or_else(|| (self.sapling_activation_height(), BlockHash([0; 32]), 0)); let cb = fake_compact_block_spending( - &self.params, + &self.network(), height, prev_hash, note, @@ -296,7 +278,7 @@ where >, > { scan_cached_blocks( - &self.params, + &self.network(), self.cache.block_source(), &mut self.db_data, from_height, @@ -316,9 +298,9 @@ impl TestState { &mut self.db_data } - /// Exposes an immutable reference to the network in use. - pub(crate) fn network(&self) -> &Network { - &self.db_data.params + /// Exposes the network in use. + pub(crate) fn network(&self) -> Network { + self.db_data.params } /// Convenience method for obtaining the Sapling activation height for the network under test. @@ -362,9 +344,10 @@ impl TestState { ReceivedNoteId, >, > { + let params = self.network(); create_spend_to_address( &mut self.db_data, - &self.params, + ¶ms, test_prover(), usk, to, @@ -397,9 +380,10 @@ impl TestState { where InputsT: InputSelector>, { + let params = self.network(); spend( &mut self.db_data, - &self.params, + ¶ms, test_prover(), input_selector, usk, @@ -430,9 +414,10 @@ impl TestState { where InputsT: InputSelector>, { + let params = self.network(); propose_transfer::<_, _, _, Infallible>( &mut self.db_data, - &self.params, + ¶ms, spend_from_account, input_selector, request, @@ -462,9 +447,10 @@ impl TestState { where InputsT: InputSelector>, { + let params = self.network(); propose_shielding::<_, _, _, Infallible>( &mut self.db_data, - &self.params, + ¶ms, input_selector, shielding_threshold, from_addrs, @@ -493,9 +479,10 @@ impl TestState { where FeeRuleT: FeeRule, { + let params = self.network(); create_proposed_transaction::<_, _, Infallible, _>( &mut self.db_data, - &self.params, + ¶ms, test_prover(), usk, ovk_policy, @@ -529,9 +516,10 @@ impl TestState { where InputsT: InputSelector>, { + let params = self.network(); shield_transparent_funds( &mut self.db_data, - &self.params, + ¶ms, test_prover(), input_selector, shielding_threshold, diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 354016fe47..5856970aea 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -391,7 +391,10 @@ mod tests { zip32::sapling::ExtendedFullViewingKey, }; - use crate::{error::SqliteClientError, wallet::scanning::priority_code, AccountId, WalletDb}; + use crate::{ + error::SqliteClientError, testing::TestBuilder, wallet::scanning::priority_code, AccountId, + WalletDb, + }; use super::{init_accounts_table, init_blocks_table, init_wallet_db}; @@ -405,9 +408,7 @@ mod tests { #[test] fn verify_schema() { - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); - init_wallet_db(&mut db_data, None).unwrap(); + let st = TestBuilder::new().build(); use regex::Regex; let re = Regex::new(r"\s+").unwrap(); @@ -560,7 +561,8 @@ mod tests { )", ]; - let mut tables_query = db_data + let mut tables_query = st + .wallet() .conn .prepare("SELECT sql FROM sqlite_schema WHERE type = 'table' ORDER BY tbl_name") .unwrap(); @@ -600,7 +602,7 @@ mod tests { AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height ) WHERE scan_queue.priority > {}", - u32::from(db_data.params.activation_height(NetworkUpgrade::Sapling).unwrap()), + u32::from(st.network().activation_height(NetworkUpgrade::Sapling).unwrap()), priority_code(&ScanPriority::Scanned), ), // v_transactions @@ -741,7 +743,8 @@ mod tests { OR sapling_received_notes.is_change = 0".to_owned(), ]; - let mut views_query = db_data + let mut views_query = st + .wallet() .conn .prepare("SELECT sql FROM sqlite_schema WHERE type = 'view' ORDER BY tbl_name") .unwrap(); diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index db1d126c74..9c254c7890 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -532,7 +532,7 @@ pub(crate) mod tests { let ufvks = [(account, usk.to_unified_full_viewing_key())] .into_iter() .collect(); - let decrypted_outputs = decrypt_transaction(st.network(), h + 1, &tx, &ufvks); + let decrypted_outputs = decrypt_transaction(&st.network(), h + 1, &tx, &ufvks); assert_eq!(decrypted_outputs.len(), 2); let mut found_tx_change_memo = false; @@ -619,7 +619,7 @@ pub(crate) mod tests { // Create a USK that doesn't exist in the wallet let acct1 = AccountId::from(1); - let usk1 = UnifiedSpendingKey::from_seed(st.network(), &[1u8; 32], acct1).unwrap(); + let usk1 = UnifiedSpendingKey::from_seed(&st.network(), &[1u8; 32], acct1).unwrap(); // Attempting to spend with a USK that is not in the wallet results in an error assert_matches!( @@ -934,7 +934,7 @@ pub(crate) mod tests { for output in tx.sapling_bundle().unwrap().shielded_outputs() { // Find the output that decrypts with the external OVK let result = try_sapling_output_recovery( - st.network(), + &st.network(), h1, &dfvk.to_ovk(Scope::External), output, diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index ff1d91c851..e7d8c7668f 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -1200,7 +1200,7 @@ mod tests { fn create_account_creates_ignored_range() { use ScanPriority::*; - let mut st = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new().build(); let sap_active = st.sapling_activation_height(); From 57a3914e3a364a5cd2f0487af4e061b50859003b Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 15 May 2023 20:44:02 +0000 Subject: [PATCH 0118/1122] zcash_primitives: Drop `byteorder::LittleEndian` usage in sighash --- .../src/transaction/sighash_v4.rs | 28 ++++++------------- .../src/transaction/sighash_v5.rs | 6 ++-- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/zcash_primitives/src/transaction/sighash_v4.rs b/zcash_primitives/src/transaction/sighash_v4.rs index b5eeed7b1c..122bdf6188 100644 --- a/zcash_primitives/src/transaction/sighash_v4.rs +++ b/zcash_primitives/src/transaction/sighash_v4.rs @@ -1,5 +1,4 @@ use blake2b_simd::{Hash as Blake2bHash, Params as Blake2bParams}; -use byteorder::{LittleEndian, WriteBytesExt}; use ff::PrimeField; use crate::consensus::BranchId; @@ -22,13 +21,6 @@ const ZCASH_JOINSPLITS_HASH_PERSONALIZATION: &[u8; 16] = b"ZcashJSplitsHash"; const ZCASH_SHIELDED_SPENDS_HASH_PERSONALIZATION: &[u8; 16] = b"ZcashSSpendsHash"; const ZCASH_SHIELDED_OUTPUTS_HASH_PERSONALIZATION: &[u8; 16] = b"ZcashSOutputHash"; -macro_rules! update_u32 { - ($h:expr, $value:expr, $tmp:expr) => { - (&mut $tmp[..4]).write_u32::($value).unwrap(); - $h.update(&$tmp[..4]); - }; -} - macro_rules! update_hash { ($h:expr, $cond:expr, $value:expr) => { if $cond { @@ -53,7 +45,7 @@ fn prevout_hash(vin: &[TxIn]) -> Blake2bHash fn sequence_hash(vin: &[TxIn]) -> Blake2bHash { let mut data = Vec::with_capacity(vin.len() * 4); for t_in in vin { - data.write_u32::(t_in.sequence).unwrap(); + data.extend_from_slice(&t_in.sequence.to_le_bytes()); } Blake2bParams::new() .hash_length(32) @@ -145,18 +137,15 @@ pub fn v4_signature_hash< if tx.version.has_overwinter() { let mut personal = [0; 16]; personal[..12].copy_from_slice(ZCASH_SIGHASH_PERSONALIZATION_PREFIX); - (&mut personal[12..]) - .write_u32::(tx.consensus_branch_id.into()) - .unwrap(); + personal[12..].copy_from_slice(&u32::from(tx.consensus_branch_id).to_le_bytes()); let mut h = Blake2bParams::new() .hash_length(32) .personal(&personal) .to_state(); - let mut tmp = [0; 8]; - update_u32!(h, tx.version.header(), tmp); - update_u32!(h, tx.version.version_group_id(), tmp); + h.update(&tx.version.header().to_le_bytes()); + h.update(&tx.version.version_group_id().to_le_bytes()); update_hash!( h, hash_type & SIGHASH_ANYONECANPAY == 0, @@ -231,12 +220,12 @@ pub fn v4_signature_hash< shielded_outputs_hash(tx.sapling_bundle.as_ref().unwrap().shielded_outputs()) ); } - update_u32!(h, tx.lock_time, tmp); - update_u32!(h, tx.expiry_height.into(), tmp); + h.update(&tx.lock_time.to_le_bytes()); + h.update(&u32::from(tx.expiry_height).to_le_bytes()); if tx.version.has_sapling() { h.update(&tx.sapling_value_balance().to_i64_le_bytes()); } - update_u32!(h, hash_type.into(), tmp); + h.update(&u32::from(hash_type).to_le_bytes()); match signable_input { SignableInput::Shielded => (), @@ -251,8 +240,7 @@ pub fn v4_signature_hash< bundle.vin[*index].prevout.write(&mut data).unwrap(); script_code.write(&mut data).unwrap(); data.extend_from_slice(&value.to_i64_le_bytes()); - data.write_u32::(bundle.vin[*index].sequence) - .unwrap(); + data.extend_from_slice(&bundle.vin[*index].sequence.to_le_bytes()); h.update(&data); } else { panic!( diff --git a/zcash_primitives/src/transaction/sighash_v5.rs b/zcash_primitives/src/transaction/sighash_v5.rs index aebe5f9193..72d15d94ff 100644 --- a/zcash_primitives/src/transaction/sighash_v5.rs +++ b/zcash_primitives/src/transaction/sighash_v5.rs @@ -1,7 +1,6 @@ use std::io::Write; use blake2b_simd::{Hash as Blake2bHash, Params, State}; -use byteorder::{LittleEndian, WriteBytesExt}; use zcash_encoding::Array; use crate::transaction::{ @@ -17,6 +16,9 @@ use crate::transaction::{ Authorization, TransactionData, TransparentDigests, TxDigests, }; +#[cfg(feature = "zfuture")] +use byteorder::WriteBytesExt; + #[cfg(feature = "zfuture")] use zcash_encoding::{CompactSize, Vector}; @@ -121,7 +123,7 @@ fn transparent_sig_digest( txin.prevout.write(&mut ch).unwrap(); ch.write_all(&value.to_i64_le_bytes()).unwrap(); script_pubkey.write(&mut ch).unwrap(); - ch.write_u32::(txin.sequence).unwrap(); + ch.write_all(&txin.sequence.to_le_bytes()).unwrap(); } let txin_sig_digest = ch.finalize(); From c8e2d81f58d4ba0193017633b983e251639b625d Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 15 May 2023 20:45:38 +0000 Subject: [PATCH 0119/1122] Render byte slices as hex more often in `Debug` impls This is more generally useful for debugging purposes than the default `Debug` impl for `&[u8]`. We also provide an alternate `Debug` impl for `legacy::Script` that parses and renders known opcodes. Note that we only parse a subset of the full opcode set. --- components/zcash_note_encryption/src/lib.rs | 23 +++++++- zcash_primitives/src/legacy.rs | 59 ++++++++++++++++++- .../src/sapling/note/nullifier.rs | 11 +++- zcash_primitives/src/sapling/redjubjub.rs | 13 +++- 4 files changed, 102 insertions(+), 4 deletions(-) diff --git a/components/zcash_note_encryption/src/lib.rs b/components/zcash_note_encryption/src/lib.rs index fb8049d40c..16c089bbab 100644 --- a/components/zcash_note_encryption/src/lib.rs +++ b/components/zcash_note_encryption/src/lib.rs @@ -19,6 +19,8 @@ #![deny(unsafe_code)] // TODO: #![deny(missing_docs)] +use core::fmt::{self, Write}; + #[cfg(feature = "alloc")] extern crate alloc; #[cfg(feature = "alloc")] @@ -72,9 +74,28 @@ impl AsRef<[u8]> for OutgoingCipherKey { /// Newtype representing the byte encoding of an [`EphemeralPublicKey`]. /// /// [`EphemeralPublicKey`]: Domain::EphemeralPublicKey -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct EphemeralKeyBytes(pub [u8; 32]); +impl fmt::Debug for EphemeralKeyBytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + struct HexFmt<'b>(&'b [u8]); + impl<'b> fmt::Debug for HexFmt<'b> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_char('"')?; + for b in self.0 { + f.write_fmt(format_args!("{:02x}", b))?; + } + f.write_char('"') + } + } + + f.debug_tuple("EphemeralKeyBytes") + .field(&HexFmt(&self.0)) + .finish() + } +} + impl AsRef<[u8]> for EphemeralKeyBytes { fn as_ref(&self) -> &[u8] { &self.0 diff --git a/zcash_primitives/src/legacy.rs b/zcash_primitives/src/legacy.rs index e1d12a766f..cdbbc25122 100644 --- a/zcash_primitives/src/legacy.rs +++ b/zcash_primitives/src/legacy.rs @@ -1,6 +1,8 @@ //! Support for legacy transparent addresses and scripts. use byteorder::{ReadBytesExt, WriteBytesExt}; + +use std::fmt; use std::io::{self, Read, Write}; use std::ops::Shl; @@ -10,6 +12,7 @@ use zcash_encoding::Vector; pub mod keys; /// Minimal subset of script opcodes. +#[derive(Debug)] enum OpCode { // push value PushData1 = 0x4c, @@ -28,10 +31,64 @@ enum OpCode { CheckSig = 0xac, } +impl OpCode { + fn parse(b: u8) -> Option { + match b { + 0x4c => Some(OpCode::PushData1), + 0x4d => Some(OpCode::PushData2), + 0x4e => Some(OpCode::PushData4), + 0x76 => Some(OpCode::Dup), + 0x87 => Some(OpCode::Equal), + 0x88 => Some(OpCode::EqualVerify), + 0xa9 => Some(OpCode::Hash160), + 0xac => Some(OpCode::CheckSig), + _ => None, + } + } +} + /// A serialized script, used inside transparent inputs and outputs of a transaction. -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Default, PartialEq, Eq)] pub struct Script(pub Vec); +impl fmt::Debug for Script { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + struct ScriptPrinter<'s>(&'s [u8]); + impl<'s> fmt::Debug for ScriptPrinter<'s> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut l = f.debug_list(); + let mut unknown: Option = None; + for b in self.0 { + if let Some(opcode) = OpCode::parse(*b) { + if let Some(s) = unknown.take() { + l.entry(&s); + } + l.entry(&opcode); + } else { + let encoded = format!("{:02x}", b); + if let Some(s) = &mut unknown { + s.push_str(&encoded); + } else { + unknown = Some(encoded); + } + } + } + l.finish() + } + } + + if f.alternate() { + f.debug_tuple("Script") + .field(&ScriptPrinter(&self.0)) + .finish() + } else { + f.debug_tuple("Script") + .field(&hex::encode(&self.0)) + .finish() + } + } +} + impl Script { pub fn read(mut reader: R) -> io::Result { let script = Vector::read(&mut reader, |r| r.read_u8())?; diff --git a/zcash_primitives/src/sapling/note/nullifier.rs b/zcash_primitives/src/sapling/note/nullifier.rs index 86f7bf946d..6fd212de47 100644 --- a/zcash_primitives/src/sapling/note/nullifier.rs +++ b/zcash_primitives/src/sapling/note/nullifier.rs @@ -1,4 +1,5 @@ use std::array::TryFromSliceError; +use std::fmt; use subtle::{Choice, ConstantTimeEq}; @@ -9,9 +10,17 @@ use crate::sapling::{ }; /// Typesafe wrapper for nullifier values. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq)] pub struct Nullifier(pub [u8; 32]); +impl fmt::Debug for Nullifier { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Nullifier") + .field(&hex::encode(self.0)) + .finish() + } +} + impl Nullifier { pub fn from_slice(bytes: &[u8]) -> Result { bytes.try_into().map(Nullifier) diff --git a/zcash_primitives/src/sapling/redjubjub.rs b/zcash_primitives/src/sapling/redjubjub.rs index 74b7b6e0d5..695098681b 100644 --- a/zcash_primitives/src/sapling/redjubjub.rs +++ b/zcash_primitives/src/sapling/redjubjub.rs @@ -7,6 +7,8 @@ use ff::{Field, PrimeField}; use group::GroupEncoding; use jubjub::{AffinePoint, ExtendedPoint, SubgroupPoint}; use rand_core::RngCore; + +use std::fmt; use std::io::{self, Read, Write}; use std::ops::{AddAssign, MulAssign, Neg}; @@ -28,12 +30,21 @@ fn h_star(a: &[u8], b: &[u8]) -> jubjub::Fr { hash_to_scalar(b"Zcash_RedJubjubH", a, b) } -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone)] pub struct Signature { rbar: [u8; 32], sbar: [u8; 32], } +impl fmt::Debug for Signature { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Signature") + .field("rbar", &hex::encode(self.rbar)) + .field("sbar", &hex::encode(self.sbar)) + .finish() + } +} + pub struct PrivateKey(pub jubjub::Fr); #[derive(Debug, Clone)] From f33c8ea418a4f99c151ade18755017238a98d5b7 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 30 Aug 2023 20:20:20 +0000 Subject: [PATCH 0120/1122] zcash_primitives: Parse the full opcode set This enhances the alternate `impl Debug for Script` to render unexpected scripts correctly. --- zcash_primitives/src/legacy.rs | 225 ++++++++++++++++++++++++++++++++- 1 file changed, 224 insertions(+), 1 deletion(-) diff --git a/zcash_primitives/src/legacy.rs b/zcash_primitives/src/legacy.rs index cdbbc25122..17ae8b0a0c 100644 --- a/zcash_primitives/src/legacy.rs +++ b/zcash_primitives/src/legacy.rs @@ -11,37 +11,260 @@ use zcash_encoding::Vector; #[cfg(feature = "transparent-inputs")] pub mod keys; -/// Minimal subset of script opcodes. +/// Defined script opcodes. +/// +/// Most of the opcodes are unused by this crate, but we define them so that the alternate +/// `Debug` impl for [`Script`] renders correctly for unexpected scripts. #[derive(Debug)] enum OpCode { // push value + Op0 = 0x00, // False PushData1 = 0x4c, PushData2 = 0x4d, PushData4 = 0x4e, + Negative1 = 0x4f, + Reserved = 0x50, + Op1 = 0x51, // True + Op2 = 0x52, + Op3 = 0x53, + Op4 = 0x54, + Op5 = 0x55, + Op6 = 0x56, + Op7 = 0x57, + Op8 = 0x58, + Op9 = 0x59, + Op10 = 0x5a, + Op11 = 0x5b, + Op12 = 0x5c, + Op13 = 0x5d, + Op14 = 0x5e, + Op15 = 0x5f, + Op16 = 0x60, + + // control + Nop = 0x61, + Ver = 0x62, + If = 0x63, + NotIf = 0x64, + VerIf = 0x65, + VerNotIf = 0x66, + Else = 0x67, + EndIf = 0x68, + Verify = 0x69, + Return = 0x6a, // stack ops + ToAltStack = 0x6b, + FromAltStack = 0x6c, + Drop2 = 0x6d, + Dup2 = 0x6e, + Dup3 = 0x6f, + Over2 = 0x70, + Rot2 = 0x71, + Swap2 = 0x72, + IfDup = 0x73, + Depth = 0x74, + Drop = 0x75, Dup = 0x76, + Nip = 0x77, + Over = 0x78, + Pick = 0x79, + Roll = 0x7a, + Rot = 0x7b, + Swap = 0x7c, + Tuck = 0x7d, + + // splice ops + Cat = 0x7e, // Disabled + Substr = 0x7f, // Disabled + Left = 0x80, // Disabled + Right = 0x81, // Disabled + Size = 0x82, // bit logic + Invert = 0x83, // Disabled + And = 0x84, // Disabled + Or = 0x85, // Disabled + Xor = 0x86, // Disabled Equal = 0x87, EqualVerify = 0x88, + Reserved1 = 0x89, + Reserved2 = 0x8a, + + // numeric + Add1 = 0x8b, + Sub1 = 0x8c, + Mul2 = 0x8d, // Disabled + Div2 = 0x8e, // Disabled + Negate = 0x8f, + Abs = 0x90, + Not = 0x91, + NotEqual0 = 0x92, + + Add = 0x93, + Sub = 0x94, + Mul = 0x95, // Disabled + Div = 0x96, // Disabled + Mod = 0x97, // Disabled + LShift = 0x98, // Disabled + RShift = 0x99, // Disabled + + BoolAnd = 0x9a, + BoolOr = 0x9b, + NumEqual = 0x9c, + NumEqualVerify = 0x9d, + NumNotEqual = 0x9e, + LessThan = 0x9f, + GreaterThan = 0xa0, + LessThanOrEqual = 0xa1, + GreaterThanOrEqual = 0xa2, + Min = 0xa3, + Max = 0xa4, + + Within = 0xa5, // crypto + Ripemd160 = 0xa6, + Sha1 = 0xa7, + Sha256 = 0xa8, Hash160 = 0xa9, + Hash256 = 0xaa, + CodeSeparator = 0xab, // Disabled CheckSig = 0xac, + CheckSigVerify = 0xad, + CheckMultisig = 0xae, + CheckMultisigVerify = 0xaf, + + // expansion + Nop1 = 0xb0, + CheckLockTimeVerify = 0xb1, + Nop3 = 0xb2, + Nop4 = 0xb3, + Nop5 = 0xb4, + Nop6 = 0xb5, + Nop7 = 0xb6, + Nop8 = 0xb7, + Nop9 = 0xb8, + Nop10 = 0xb9, + + InvalidOpCode = 0xff, } impl OpCode { fn parse(b: u8) -> Option { match b { + 0x00 => Some(OpCode::Op0), 0x4c => Some(OpCode::PushData1), 0x4d => Some(OpCode::PushData2), 0x4e => Some(OpCode::PushData4), + 0x4f => Some(OpCode::Negative1), + 0x50 => Some(OpCode::Reserved), + 0x51 => Some(OpCode::Op1), + 0x52 => Some(OpCode::Op2), + 0x53 => Some(OpCode::Op3), + 0x54 => Some(OpCode::Op4), + 0x55 => Some(OpCode::Op5), + 0x56 => Some(OpCode::Op6), + 0x57 => Some(OpCode::Op7), + 0x58 => Some(OpCode::Op8), + 0x59 => Some(OpCode::Op9), + 0x5a => Some(OpCode::Op10), + 0x5b => Some(OpCode::Op11), + 0x5c => Some(OpCode::Op12), + 0x5d => Some(OpCode::Op13), + 0x5e => Some(OpCode::Op14), + 0x5f => Some(OpCode::Op15), + 0x60 => Some(OpCode::Op16), + 0x61 => Some(OpCode::Nop), + 0x62 => Some(OpCode::Ver), + 0x63 => Some(OpCode::If), + 0x64 => Some(OpCode::NotIf), + 0x65 => Some(OpCode::VerIf), + 0x66 => Some(OpCode::VerNotIf), + 0x67 => Some(OpCode::Else), + 0x68 => Some(OpCode::EndIf), + 0x69 => Some(OpCode::Verify), + 0x6a => Some(OpCode::Return), + 0x6b => Some(OpCode::ToAltStack), + 0x6c => Some(OpCode::FromAltStack), + 0x6d => Some(OpCode::Drop2), + 0x6e => Some(OpCode::Dup2), + 0x6f => Some(OpCode::Dup3), + 0x70 => Some(OpCode::Over2), + 0x71 => Some(OpCode::Rot2), + 0x72 => Some(OpCode::Swap2), + 0x73 => Some(OpCode::IfDup), + 0x74 => Some(OpCode::Depth), + 0x75 => Some(OpCode::Drop), 0x76 => Some(OpCode::Dup), + 0x77 => Some(OpCode::Nip), + 0x78 => Some(OpCode::Over), + 0x79 => Some(OpCode::Pick), + 0x7a => Some(OpCode::Roll), + 0x7b => Some(OpCode::Rot), + 0x7c => Some(OpCode::Swap), + 0x7d => Some(OpCode::Tuck), + 0x7e => Some(OpCode::Cat), + 0x7f => Some(OpCode::Substr), + 0x80 => Some(OpCode::Left), + 0x81 => Some(OpCode::Right), + 0x82 => Some(OpCode::Size), + 0x83 => Some(OpCode::Invert), + 0x84 => Some(OpCode::And), + 0x85 => Some(OpCode::Or), + 0x86 => Some(OpCode::Xor), 0x87 => Some(OpCode::Equal), 0x88 => Some(OpCode::EqualVerify), + 0x89 => Some(OpCode::Reserved1), + 0x8a => Some(OpCode::Reserved2), + 0x8b => Some(OpCode::Add1), + 0x8c => Some(OpCode::Sub1), + 0x8d => Some(OpCode::Mul2), + 0x8e => Some(OpCode::Div2), + 0x8f => Some(OpCode::Negate), + 0x90 => Some(OpCode::Abs), + 0x91 => Some(OpCode::Not), + 0x92 => Some(OpCode::NotEqual0), + 0x93 => Some(OpCode::Add), + 0x94 => Some(OpCode::Sub), + 0x95 => Some(OpCode::Mul), + 0x96 => Some(OpCode::Div), + 0x97 => Some(OpCode::Mod), + 0x98 => Some(OpCode::LShift), + 0x99 => Some(OpCode::RShift), + 0x9a => Some(OpCode::BoolAnd), + 0x9b => Some(OpCode::BoolOr), + 0x9c => Some(OpCode::NumEqual), + 0x9d => Some(OpCode::NumEqualVerify), + 0x9e => Some(OpCode::NumNotEqual), + 0x9f => Some(OpCode::LessThan), + 0xa0 => Some(OpCode::GreaterThan), + 0xa1 => Some(OpCode::LessThanOrEqual), + 0xa2 => Some(OpCode::GreaterThanOrEqual), + 0xa3 => Some(OpCode::Min), + 0xa4 => Some(OpCode::Max), + 0xa5 => Some(OpCode::Within), + 0xa6 => Some(OpCode::Ripemd160), + 0xa7 => Some(OpCode::Sha1), + 0xa8 => Some(OpCode::Sha256), 0xa9 => Some(OpCode::Hash160), + 0xaa => Some(OpCode::Hash256), + 0xab => Some(OpCode::CodeSeparator), 0xac => Some(OpCode::CheckSig), + 0xad => Some(OpCode::CheckSigVerify), + 0xae => Some(OpCode::CheckMultisig), + 0xaf => Some(OpCode::CheckMultisigVerify), + 0xb0 => Some(OpCode::Nop1), + 0xb1 => Some(OpCode::CheckLockTimeVerify), + 0xb2 => Some(OpCode::Nop3), + 0xb3 => Some(OpCode::Nop4), + 0xb4 => Some(OpCode::Nop5), + 0xb5 => Some(OpCode::Nop6), + 0xb6 => Some(OpCode::Nop7), + 0xb7 => Some(OpCode::Nop8), + 0xb8 => Some(OpCode::Nop9), + 0xb9 => Some(OpCode::Nop10), + 0xff => Some(OpCode::InvalidOpCode), _ => None, } } From bcafb7d4b40ae742abcfccbf1e4439fd725d4f3e Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 31 Aug 2023 14:18:30 -0600 Subject: [PATCH 0121/1122] zcash_client_sqlite: Prevent scan range fragmentation. Fixes #922 Co-authored-by: Sean Bowe --- zcash_client_sqlite/src/wallet/scanning.rs | 51 ++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index e7d8c7668f..5b24cf3dc0 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -455,7 +455,7 @@ pub(crate) fn replace_queue_entries( WHERE ( -- the start is contained within the range :start >= block_range_start - AND :start < block_range_end + AND :start <= block_range_end ) OR ( -- the end is contained within the range @@ -735,7 +735,7 @@ pub(crate) fn update_chain_tip( #[cfg(test)] mod tests { - use std::ops::Range; + use std::{mem::replace, ops::Range}; use incrementalmerkletree::{Hashable, Level}; use secrecy::Secret; @@ -754,7 +754,10 @@ mod tests { use crate::{ testing::{birthday_at_sapling_activation, AddressType, TestBuilder}, - wallet::{init::init_blocks_table, scanning::suggest_scan_ranges}, + wallet::{ + init::init_blocks_table, + scanning::{insert_queue_entries, replace_queue_entries, suggest_scan_ranges}, + }, }; use super::{join_nonoverlapping, Joined, RangeOrdering, SpanningTree}; @@ -1295,4 +1298,46 @@ mod tests { let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); assert_eq!(actual, expected); } + + #[test] + fn replace_queue_entries_merges_previous_range() { + use ScanPriority::*; + + let mut st = TestBuilder::new().build(); + + let ranges = vec![ + scan_range(150..200, ChainTip), + scan_range(100..150, Scanned), + scan_range(0..100, Ignored), + ]; + + { + let tx = st.wallet_mut().conn.transaction().unwrap(); + insert_queue_entries(&tx, ranges.iter()).unwrap(); + tx.commit().unwrap(); + } + + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, ranges); + + { + let tx = st.wallet_mut().conn.transaction().unwrap(); + replace_queue_entries( + &tx, + &(BlockHeight::from(150)..BlockHeight::from(160)), + vec![scan_range(150..160, Scanned)].into_iter(), + ) + .unwrap(); + tx.commit().unwrap(); + } + + let expected = vec![ + scan_range(160..200, ChainTip), + scan_range(100..160, Scanned), + scan_range(0..100, Ignored), + ]; + + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, expected); + } } From 3ad07cad9e0064cac77ef3d3812d0b80ff303b42 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 31 Aug 2023 20:24:46 +0000 Subject: [PATCH 0122/1122] zcash_client_backend: Reduce scope of `lightwalletd-tonic` feature flag The `zcash_client_backend::proto::service::compact_tx_streamer_client` is now the only module controlled by that feature flag, exposing the service types for use by parsers. --- zcash_client_backend/CHANGELOG.md | 8 ++++++-- zcash_client_backend/build.rs | 7 ++++--- zcash_client_backend/src/proto.rs | 1 - zcash_client_backend/src/proto/service.rs | 1 + 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 8dfa7c49ba..4a0fc7e2b0 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -87,8 +87,12 @@ and this library adheres to Rust's notion of method now takes an optional `BlockMetadata` argument instead of a base commitment tree and incremental witnesses for each previously-known note. In addition, the return type has now been updated to return a `Result`. -- `proto/service.proto` has been updated to include the new GRPC endpoints - supported by lightwalletd v0.4.15 +- `zcash_client_backend::proto::service`: + - The module is no longer behind the `lightwalletd-tonic` feature flag; that + now only gates the `service::compact_tx_streamer_client` submodule. This + exposes the service types to parse messages received by other gRPC clients. + - The module has been updated to include the new gRPC endpoints supported by + `lightwalletd` v0.4.15. ### Removed - `zcash_client_backend::data_api`: diff --git a/zcash_client_backend/build.rs b/zcash_client_backend/build.rs index fdc201f572..335848d445 100644 --- a/zcash_client_backend/build.rs +++ b/zcash_client_backend/build.rs @@ -4,8 +4,6 @@ use std::io; use std::path::{Path, PathBuf}; const COMPACT_FORMATS_PROTO: &str = "proto/compact_formats.proto"; - -#[cfg(feature = "lightwalletd-tonic")] const SERVICE_PROTO: &str = "proto/service.proto"; fn main() -> io::Result<()> { @@ -40,11 +38,14 @@ fn build() -> io::Result<()> { "src/proto/compact_formats.rs", )?; - #[cfg(feature = "lightwalletd-tonic")] { // Build the gRPC types and client. tonic_build::configure() .build_server(false) + .client_mod_attribute( + "cash.z.wallet.sdk.rpc", + r#"#[cfg(feature = "lightwalletd-tonic")]"#, + ) .extern_path( ".cash.z.wallet.sdk.rpc.ChainMetadata", "crate::proto::compact_formats::ChainMetadata", diff --git a/zcash_client_backend/src/proto.rs b/zcash_client_backend/src/proto.rs index 761a5affd0..d6cc593eb9 100644 --- a/zcash_client_backend/src/proto.rs +++ b/zcash_client_backend/src/proto.rs @@ -14,7 +14,6 @@ use zcash_note_encryption::{EphemeralKeyBytes, COMPACT_NOTE_SIZE}; #[allow(clippy::derive_partial_eq_without_eq)] pub mod compact_formats; -#[cfg(feature = "lightwalletd-tonic")] #[rustfmt::skip] #[allow(unknown_lints)] #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/zcash_client_backend/src/proto/service.rs b/zcash_client_backend/src/proto/service.rs index 677e43e307..5e2d1b59a4 100644 --- a/zcash_client_backend/src/proto/service.rs +++ b/zcash_client_backend/src/proto/service.rs @@ -275,6 +275,7 @@ impl ShieldedProtocol { } } /// Generated client implementations. +#[cfg(feature = "lightwalletd-tonic")] pub mod compact_tx_streamer_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] use tonic::codegen::*; From 3272f129e23ef3b14950b2f542814ebdbd7cdb6a Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 31 Aug 2023 20:32:42 +0000 Subject: [PATCH 0123/1122] zcash_client_backend: Remove now-superfluous braces from build script --- zcash_client_backend/build.rs | 74 +++++++++++++++++------------------ 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/zcash_client_backend/build.rs b/zcash_client_backend/build.rs index 335848d445..ae7db927ff 100644 --- a/zcash_client_backend/build.rs +++ b/zcash_client_backend/build.rs @@ -38,45 +38,43 @@ fn build() -> io::Result<()> { "src/proto/compact_formats.rs", )?; - { - // Build the gRPC types and client. - tonic_build::configure() - .build_server(false) - .client_mod_attribute( - "cash.z.wallet.sdk.rpc", - r#"#[cfg(feature = "lightwalletd-tonic")]"#, - ) - .extern_path( - ".cash.z.wallet.sdk.rpc.ChainMetadata", - "crate::proto::compact_formats::ChainMetadata", - ) - .extern_path( - ".cash.z.wallet.sdk.rpc.CompactBlock", - "crate::proto::compact_formats::CompactBlock", - ) - .extern_path( - ".cash.z.wallet.sdk.rpc.CompactTx", - "crate::proto::compact_formats::CompactTx", - ) - .extern_path( - ".cash.z.wallet.sdk.rpc.CompactSaplingSpend", - "crate::proto::compact_formats::CompactSaplingSpend", - ) - .extern_path( - ".cash.z.wallet.sdk.rpc.CompactSaplingOutput", - "crate::proto::compact_formats::CompactSaplingOutput", - ) - .extern_path( - ".cash.z.wallet.sdk.rpc.CompactOrchardAction", - "crate::proto::compact_formats::CompactOrchardAction", - ) - .compile(&[SERVICE_PROTO], &["proto/"])?; + // Build the gRPC types and client. + tonic_build::configure() + .build_server(false) + .client_mod_attribute( + "cash.z.wallet.sdk.rpc", + r#"#[cfg(feature = "lightwalletd-tonic")]"#, + ) + .extern_path( + ".cash.z.wallet.sdk.rpc.ChainMetadata", + "crate::proto::compact_formats::ChainMetadata", + ) + .extern_path( + ".cash.z.wallet.sdk.rpc.CompactBlock", + "crate::proto::compact_formats::CompactBlock", + ) + .extern_path( + ".cash.z.wallet.sdk.rpc.CompactTx", + "crate::proto::compact_formats::CompactTx", + ) + .extern_path( + ".cash.z.wallet.sdk.rpc.CompactSaplingSpend", + "crate::proto::compact_formats::CompactSaplingSpend", + ) + .extern_path( + ".cash.z.wallet.sdk.rpc.CompactSaplingOutput", + "crate::proto::compact_formats::CompactSaplingOutput", + ) + .extern_path( + ".cash.z.wallet.sdk.rpc.CompactOrchardAction", + "crate::proto::compact_formats::CompactOrchardAction", + ) + .compile(&[SERVICE_PROTO], &["proto/"])?; - // Copy the generated types into the source tree so changes can be committed. The - // file has the same name as for the compact format types because they have the - // same package, but we've set things up so this only contains the service types. - fs::copy(out.join("cash.z.wallet.sdk.rpc.rs"), "src/proto/service.rs")?; - } + // Copy the generated types into the source tree so changes can be committed. The + // file has the same name as for the compact format types because they have the + // same package, but we've set things up so this only contains the service types. + fs::copy(out.join("cash.z.wallet.sdk.rpc.rs"), "src/proto/service.rs")?; Ok(()) } From 4fe9d61959744efa822009e40e68c5cda25a02a4 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 31 Aug 2023 15:07:07 -0600 Subject: [PATCH 0124/1122] zcash_client_sqlite: Fix a comment on range inclusion. Co-authored-by: str4d --- zcash_client_sqlite/src/wallet/scanning.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 5b24cf3dc0..a857d82da7 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -453,7 +453,7 @@ pub(crate) fn replace_queue_entries( "SELECT block_range_start, block_range_end, priority FROM scan_queue WHERE ( - -- the start is contained within the range + -- the start is contained within or adjacent to the range :start >= block_range_start AND :start <= block_range_end ) From fb8c3a178c166534bd45dd80206b976947125486 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 31 Aug 2023 17:52:29 -0600 Subject: [PATCH 0125/1122] zcash_client_sqlite: Ensure that fragmentation also can't occur at scan range ends. --- zcash_client_sqlite/src/wallet/scanning.rs | 46 +++++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index a857d82da7..8bdac31a9d 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -458,8 +458,8 @@ pub(crate) fn replace_queue_entries( AND :start <= block_range_end ) OR ( - -- the end is contained within the range - :end > block_range_start + -- the end is contained within or adjacent to the range + :end >= block_range_start AND :end <= block_range_end ) OR ( @@ -1340,4 +1340,46 @@ mod tests { let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); assert_eq!(actual, expected); } + + #[test] + fn replace_queue_entries_merges_subsequent_range() { + use ScanPriority::*; + + let mut st = TestBuilder::new().build(); + + let ranges = vec![ + scan_range(150..200, ChainTip), + scan_range(100..150, Scanned), + scan_range(0..100, Ignored), + ]; + + { + let tx = st.wallet_mut().conn.transaction().unwrap(); + insert_queue_entries(&tx, ranges.iter()).unwrap(); + tx.commit().unwrap(); + } + + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, ranges); + + { + let tx = st.wallet_mut().conn.transaction().unwrap(); + replace_queue_entries( + &tx, + &(BlockHeight::from(90)..BlockHeight::from(100)), + vec![scan_range(90..100, Scanned)].into_iter(), + ) + .unwrap(); + tx.commit().unwrap(); + } + + let expected = vec![ + scan_range(150..200, ChainTip), + scan_range(90..150, Scanned), + scan_range(0..90, Ignored), + ]; + + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, expected); + } } From 8bcc3c1cc16177fd33b976c638db47deb78f3b60 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 31 Aug 2023 17:56:04 -0600 Subject: [PATCH 0126/1122] Fix unused imports. --- zcash_client_sqlite/src/testing.rs | 5 +++-- zcash_client_sqlite/src/wallet.rs | 3 +-- zcash_client_sqlite/src/wallet/scanning.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index b98e546ec2..a309dcba95 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -38,7 +38,6 @@ use zcash_note_encryption::Domain; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight, Network, NetworkUpgrade, Parameters}, - legacy::TransparentAddress, memo::MemoBytes, sapling::{ note_encryption::{sapling_note_encryption, SaplingDomain}, @@ -66,7 +65,9 @@ use super::BlockDb; #[cfg(feature = "transparent-inputs")] use { zcash_client_backend::data_api::wallet::{propose_shielding, shield_transparent_funds}, - zcash_primitives::transaction::components::amount::NonNegativeAmount, + zcash_primitives::{ + legacy::TransparentAddress, transaction::components::amount::NonNegativeAmount, + }, }; #[cfg(feature = "unstable")] diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index e1f8e19c36..ef34083b76 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1571,8 +1571,6 @@ pub(crate) fn prune_nullifier_map( mod tests { use std::num::NonZeroU32; - use secrecy::Secret; - use zcash_primitives::transaction::components::Amount; use zcash_client_backend::data_api::WalletRead; @@ -1586,6 +1584,7 @@ mod tests { #[cfg(feature = "transparent-inputs")] use { + secrecy::Secret, zcash_client_backend::{ data_api::WalletWrite, encoding::AddressCodec, wallet::WalletTransparentOutput, }, diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 8bdac31a9d..b2a38d0957 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -735,7 +735,7 @@ pub(crate) fn update_chain_tip( #[cfg(test)] mod tests { - use std::{mem::replace, ops::Range}; + use std::ops::Range; use incrementalmerkletree::{Hashable, Level}; use secrecy::Secret; From c99c0fc884065da7f14951f2997b0d2a45f80ed3 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 16 Aug 2023 11:15:10 -0600 Subject: [PATCH 0127/1122] zcash_client_sqlite: Enable forced re-scanning of previously scanned ranges. When `force_rescans` is set to `true` in a call to `replace_queue_entries`, previously scanned ranges will have their existing priority overwritten by the scan priority for a provided range; otherwise, the existing scan priority dominance rule continues to be enforced. This enables us to require previously scanned ranges be re-scanned without interfering with higher-priority scan operations. --- zcash_client_sqlite/src/wallet.rs | 2 +- zcash_client_sqlite/src/wallet/scanning.rs | 157 +++++++++++++++------ 2 files changed, 116 insertions(+), 43 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index ef34083b76..1b3f1a12a3 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -949,7 +949,7 @@ pub(crate) fn truncate_to_height( // Prioritize the range starting at the height we just rewound to for verification let query_range = block_height..(block_height + VERIFY_LOOKAHEAD); let scan_range = ScanRange::from_parts(query_range.clone(), ScanPriority::Verify); - replace_queue_entries(conn, &query_range, Some(scan_range).into_iter())?; + replace_queue_entries(conn, &query_range, Some(scan_range).into_iter(), false)?; } Ok(()) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index b2a38d0957..192d7b457f 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -17,18 +17,42 @@ use crate::{PRUNING_DEPTH, VERIFY_LOOKAHEAD}; use super::block_height_extrema; #[derive(Debug, Clone, Copy)] -enum Insert { +enum InsertOn { Left, Right, } +struct Insert { + on: InsertOn, + force_rescan: bool, +} + +impl Insert { + fn left(force_rescan: bool) -> Self { + Insert { + on: InsertOn::Left, + force_rescan, + } + } + + fn right(force_rescan: bool) -> Self { + Insert { + on: InsertOn::Right, + force_rescan, + } + } +} + impl Not for Insert { type Output = Self; fn not(self) -> Self::Output { - match self { - Insert::Left => Insert::Right, - Insert::Right => Insert::Left, + Insert { + on: match self.on { + InsertOn::Left => InsertOn::Right, + InsertOn::Right => InsertOn::Left, + }, + force_rescan: self.force_rescan, } } } @@ -42,9 +66,9 @@ enum Dominance { impl From for Dominance { fn from(value: Insert) -> Self { - match value { - Insert::Left => Dominance::Left, - Insert::Right => Dominance::Right, + match value.on { + InsertOn::Left => Dominance::Left, + InsertOn::Right => Dominance::Right, } } } @@ -115,7 +139,7 @@ fn dominance(current: &ScanPriority, inserted: &ScanPriority, insert: Insert) -> match (current.cmp(inserted), (current, inserted)) { (Ordering::Equal, _) => Dominance::Equal, (_, (_, ScanPriority::Verify | ScanPriority::Scanned)) => Dominance::from(insert), - (_, (ScanPriority::Scanned, _)) => Dominance::from(!insert), + (_, (ScanPriority::Scanned, _)) if !insert.force_rescan => Dominance::from(!insert), (Ordering::Less, _) => Dominance::from(insert), (Ordering::Greater, _) => Dominance::from(!insert), } @@ -197,7 +221,7 @@ fn join_nonoverlapping(left: ScanRange, right: ScanRange) -> Joined { } } -fn insert(current: ScanRange, to_insert: ScanRange) -> Joined { +fn insert(current: ScanRange, to_insert: ScanRange, force_rescans: bool) -> Joined { fn join_overlapping(left: ScanRange, right: ScanRange, insert: Insert) -> Joined { assert!( left.block_range().start <= right.block_range().start @@ -205,9 +229,9 @@ fn insert(current: ScanRange, to_insert: ScanRange) -> Joined { ); // recompute the range dominance based upon the queue entry priorities - let dominance = match insert { - Insert::Left => dominance(&right.priority(), &left.priority(), insert), - Insert::Right => dominance(&left.priority(), &right.priority(), insert), + let dominance = match insert.on { + InsertOn::Left => dominance(&right.priority(), &left.priority(), insert), + InsertOn::Right => dominance(&left.priority(), &right.priority(), insert), }; match dominance { @@ -237,15 +261,23 @@ fn insert(current: ScanRange, to_insert: ScanRange) -> Joined { use RangeOrdering::*; match RangeOrdering::cmp(to_insert.block_range(), current.block_range()) { LeftFirstDisjoint => join_nonoverlapping(to_insert, current), - LeftFirstOverlap | RightContained => join_overlapping(to_insert, current, Insert::Left), + LeftFirstOverlap | RightContained => { + join_overlapping(to_insert, current, Insert::left(force_rescans)) + } Equal => Joined::One(ScanRange::from_parts( to_insert.block_range().clone(), - match dominance(¤t.priority(), &to_insert.priority(), Insert::Right) { + match dominance( + ¤t.priority(), + &to_insert.priority(), + Insert::right(force_rescans), + ) { Dominance::Left | Dominance::Equal => current.priority(), Dominance::Right => to_insert.priority(), }, )), - RightFirstOverlap | LeftContained => join_overlapping(current, to_insert, Insert::Right), + RightFirstOverlap | LeftContained => { + join_overlapping(current, to_insert, Insert::right(force_rescans)) + } RightFirstDisjoint => join_nonoverlapping(current, to_insert), } } @@ -294,9 +326,9 @@ impl SpanningTree { to_insert: ScanRange, insert: Insert, ) -> Self { - let (left, right) = match insert { - Insert::Left => (Box::new(left.insert(to_insert)), right), - Insert::Right => (left, Box::new(right.insert(to_insert))), + let (left, right) = match insert.on { + InsertOn::Left => (Box::new(left.insert(to_insert, insert.force_rescan)), right), + InsertOn::Right => (left, Box::new(right.insert(to_insert, insert.force_rescan))), }; SpanningTree::Parent { span: left.span().start..right.span().end, @@ -305,12 +337,18 @@ impl SpanningTree { } } - fn from_split(left: Self, right: Self, to_insert: ScanRange, split_point: BlockHeight) -> Self { + fn from_split( + left: Self, + right: Self, + to_insert: ScanRange, + split_point: BlockHeight, + force_rescans: bool, + ) -> Self { let (l_insert, r_insert) = to_insert .split_at(split_point) .expect("Split point is within the range of to_insert"); - let left = Box::new(left.insert(l_insert)); - let right = Box::new(right.insert(r_insert)); + let left = Box::new(left.insert(l_insert, force_rescans)); + let right = Box::new(right.insert(r_insert, force_rescans)); SpanningTree::Parent { span: left.span().start..right.span().end, left, @@ -318,9 +356,9 @@ impl SpanningTree { } } - fn insert(self, to_insert: ScanRange) -> Self { + fn insert(self, to_insert: ScanRange, force_rescans: bool) -> Self { match self { - SpanningTree::Leaf(cur) => Self::from_joined(insert(cur, to_insert)), + SpanningTree::Leaf(cur) => Self::from_joined(insert(cur, to_insert, force_rescans)), SpanningTree::Parent { span, left, right } => { // This algorithm always preserves the existing partition point, and does not do // any rebalancing or unification of ranges within the tree. This should be okay @@ -331,15 +369,15 @@ impl SpanningTree { match RangeOrdering::cmp(&span, to_insert.block_range()) { LeftFirstDisjoint => { // extend the right-hand branch - Self::from_insert(left, right, to_insert, Insert::Right) + Self::from_insert(left, right, to_insert, Insert::right(force_rescans)) } LeftFirstOverlap => { let split_point = left.span().end; if split_point > to_insert.block_range().start { - Self::from_split(*left, *right, to_insert, split_point) + Self::from_split(*left, *right, to_insert, split_point, force_rescans) } else { // to_insert is fully contained in or equals the right child - Self::from_insert(left, right, to_insert, Insert::Right) + Self::from_insert(left, right, to_insert, Insert::right(force_rescans)) } } RightContained => { @@ -348,42 +386,42 @@ impl SpanningTree { let split_point = left.span().end; if to_insert.block_range().start >= split_point { // to_insert is fully contained in the right - Self::from_insert(left, right, to_insert, Insert::Right) + Self::from_insert(left, right, to_insert, Insert::right(force_rescans)) } else if to_insert.block_range().end <= split_point { // to_insert is fully contained in the left - Self::from_insert(left, right, to_insert, Insert::Left) + Self::from_insert(left, right, to_insert, Insert::left(force_rescans)) } else { // to_insert must be split. - Self::from_split(*left, *right, to_insert, split_point) + Self::from_split(*left, *right, to_insert, split_point, force_rescans) } } Equal => { let split_point = left.span().end; if split_point > to_insert.block_range().start { - Self::from_split(*left, *right, to_insert, split_point) + Self::from_split(*left, *right, to_insert, split_point, force_rescans) } else { // to_insert is fully contained in the right subtree - right.insert(to_insert) + right.insert(to_insert, force_rescans) } } LeftContained => { // the current span is fully contained within to_insert, so we will extend // or overwrite both sides let split_point = left.span().end; - Self::from_split(*left, *right, to_insert, split_point) + Self::from_split(*left, *right, to_insert, split_point, force_rescans) } RightFirstOverlap => { let split_point = left.span().end; if split_point < to_insert.block_range().end { - Self::from_split(*left, *right, to_insert, split_point) + Self::from_split(*left, *right, to_insert, split_point, force_rescans) } else { // to_insert is fully contained in or equals the left child - Self::from_insert(left, right, to_insert, Insert::Left) + Self::from_insert(left, right, to_insert, Insert::left(force_rescans)) } } RightFirstDisjoint => { // extend the left-hand branch - Self::from_insert(left, right, to_insert, Insert::Left) + Self::from_insert(left, right, to_insert, Insert::left(force_rescans)) } } } @@ -447,6 +485,7 @@ pub(crate) fn replace_queue_entries( conn: &rusqlite::Transaction<'_>, query_range: &Range, entries: impl Iterator, + force_rescans: bool, ) -> Result<(), SqliteClientError> { let (to_create, to_delete_ends) = { let mut suggested_stmt = conn.prepare_cached( @@ -499,7 +538,7 @@ pub(crate) fn replace_queue_entries( ); to_delete_ends.push(Value::from(u32::from(entry.block_range().end))); to_create = if let Some(cur) = to_create { - Some(cur.insert(entry)) + Some(cur.insert(entry, force_rescans)) } else { Some(SpanningTree::Leaf(entry)) }; @@ -509,7 +548,7 @@ pub(crate) fn replace_queue_entries( // start with the scanned range. for entry in entries { to_create = if let Some(cur) = to_create { - Some(cur.insert(entry)) + Some(cur.insert(entry, force_rescans)) } else { Some(SpanningTree::Leaf(entry)) }; @@ -611,6 +650,7 @@ pub(crate) fn scan_complete( conn, &query_range, Some(scanned).into_iter().chain(extensions.into_iter()), + false, )?; Ok(()) @@ -714,6 +754,7 @@ pub(crate) fn update_chain_tip( conn, &query_range, shard_entry.into_iter().chain(tip_entry.into_iter()), + false, )?; } else { // If we have neither shard data nor any existing block data in the database, we should also @@ -904,7 +945,7 @@ mod tests { let scan_range = scan_range(range.clone(), *priority); match acc { None => Some(SpanningTree::Leaf(scan_range)), - Some(t) => Some(t.insert(scan_range)), + Some(t) => Some(t.insert(scan_range, false)), } }) } @@ -1035,7 +1076,7 @@ mod tests { // a `ChainTip` insertion should not overwrite a scanned range. let mut t = spanning_tree(&[(0..3, ChainTip), (3..5, Scanned), (5..7, ChainTip)]).unwrap(); - t = t.insert(scan_range(0..7, ChainTip)); + t = t.insert(scan_range(0..7, ChainTip), false); assert_eq!( t.into_vec(), vec![ @@ -1054,7 +1095,7 @@ mod tests { scan_range(280310..280320, Scanned) ] ); - t = t.insert(scan_range(280300..280340, ChainTip)); + t = t.insert(scan_range(280300..280340, ChainTip), false); assert_eq!( t.into_vec(), vec![ @@ -1077,12 +1118,42 @@ mod tests { ]) .unwrap(); - t = t.insert(scan_range(0..3, Scanned)); - t = t.insert(scan_range(5..8, Scanned)); + t = t.insert(scan_range(0..3, Scanned), false); + t = t.insert(scan_range(5..8, Scanned), false); assert_eq!(t.into_vec(), vec![scan_range(0..10, Scanned)]); } + #[test] + fn spanning_tree_force_rescans() { + use ScanPriority::*; + + let mut t = spanning_tree(&[ + (0..3, Historic), + (3..5, Scanned), + (5..7, ChainTip), + (7..10, Scanned), + ]) + .unwrap(); + + t = t.insert(scan_range(4..9, OpenAdjacent), true); + + let expected = vec![ + scan_range(0..3, Historic), + scan_range(3..4, Scanned), + scan_range(4..5, OpenAdjacent), + scan_range(5..7, ChainTip), + scan_range(7..9, OpenAdjacent), + scan_range(9..10, Scanned), + ]; + assert_eq!(t.clone().into_vec(), expected); + + // An insert of an ignored range should not override a scanned range; the existing + // priority should prevail, and so the expected state of the tree is unchanged. + t = t.insert(scan_range(2..5, Ignored), true); + assert_eq!(t.into_vec(), expected); + } + #[test] fn scan_complete() { use ScanPriority::*; @@ -1326,6 +1397,7 @@ mod tests { &tx, &(BlockHeight::from(150)..BlockHeight::from(160)), vec![scan_range(150..160, Scanned)].into_iter(), + false, ) .unwrap(); tx.commit().unwrap(); @@ -1368,6 +1440,7 @@ mod tests { &tx, &(BlockHeight::from(90)..BlockHeight::from(100)), vec![scan_range(90..100, Scanned)].into_iter(), + false, ) .unwrap(); tx.commit().unwrap(); From 459dc49b544591a923f50bda2a39f31540994f92 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 16 Aug 2023 11:15:10 -0600 Subject: [PATCH 0128/1122] zcash_client_backend: Accept account birthday as a parameter to `create_account` This also removes the zcash_client_sqlite-specific database initialization procedures in favor of a standardized approach using the methods available via the data access API. --- zcash_client_backend/CHANGELOG.md | 12 + zcash_client_backend/Cargo.toml | 2 +- zcash_client_backend/src/data_api.rs | 119 +++++- zcash_client_backend/src/proto.rs | 18 +- zcash_client_sqlite/CHANGELOG.md | 7 + zcash_client_sqlite/src/chain.rs | 32 +- zcash_client_sqlite/src/error.rs | 9 + zcash_client_sqlite/src/lib.rs | 32 +- zcash_client_sqlite/src/testing.rs | 13 +- zcash_client_sqlite/src/wallet.rs | 146 +++++-- zcash_client_sqlite/src/wallet/init.rs | 398 ++---------------- .../src/wallet/init/migrations.rs | 6 + .../init/migrations/add_account_birthdays.rs | 75 ++++ .../wallet/init/migrations/addresses_table.rs | 20 +- .../v_sapling_shard_unscanned_ranges.rs | 15 +- zcash_client_sqlite/src/wallet/sapling.rs | 115 ++--- zcash_client_sqlite/src/wallet/scanning.rs | 198 +++++---- 17 files changed, 618 insertions(+), 599 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/add_account_birthdays.rs diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 4a0fc7e2b0..b06c540af7 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -6,6 +6,17 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +### Notable Changes + +- `zcash_client_backend` now supports out-of-order scanning of blockchain history. +- This release of `zcash_client_backend` defines the concept of an account + birthday. The account birthday is defined as the minimum height among blocks + to be scanned when recovering an account. +- Account creation now requires the caller to provide account birthday information, + including the state of the note commitment tree at the end of the block prior to + the birthday height. + ### Added - `impl Eq for zcash_client_backend::address::RecipientAddress` - `impl Eq for zcash_client_backend::zip321::{Payment, TransactionRequest}` @@ -39,6 +50,7 @@ and this library adheres to Rust's notion of - `zcash_client_backend::data_api`: - `WalletRead::TxRef` has been removed in favor of consistently using `TxId` instead. - `WalletRead::get_transaction` now takes a `TxId` as its argument. + - `WalletRead::create_account` now takes an additional `birthday` argument. - `WalletWrite::{store_decrypted_tx, store_sent_tx}` now return `Result<(), Self::Error>` as the `WalletRead::TxRef` associated type has been removed. Use `WalletRead::get_transaction` with the transaction's `TxId` instead. diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 3f47a951cd..8e6d876ffa 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -45,6 +45,7 @@ memuse = "0.2" tracing = "0.1" # - Protobuf interfaces and gRPC bindings +hex = "0.4" prost = "0.11" tonic = { version = "0.9", optional = true } @@ -80,7 +81,6 @@ which = "4" [dev-dependencies] assert_matches = "1.5" gumdrop = "0.8" -hex = "0.4" jubjub = "0.10" proptest = "1.0.0" rand_core = "0.6" diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index cda26813b9..9909ace188 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -1,8 +1,9 @@ //! Interfaces for wallet data persistence & low-level wallet utilities. -use std::collections::HashMap; use std::fmt::Debug; +use std::io; use std::num::NonZeroU32; +use std::{collections::HashMap, num::TryFromIntError}; use incrementalmerkletree::{frontier::Frontier, Retention}; use secrecy::SecretVec; @@ -24,6 +25,7 @@ use crate::{ address::{AddressMetadata, UnifiedAddress}, decrypt::DecryptedOutput, keys::{UnifiedFullViewingKey, UnifiedSpendingKey}, + proto::service::TreeState, wallet::{ReceivedSaplingNote, WalletTransparentOutput, WalletTx}, }; @@ -472,25 +474,77 @@ impl SentTransactionOutput { pub struct AccountBirthday { height: BlockHeight, sapling_frontier: Frontier, + recover_until: Option, +} + +/// Errors that can occur in the construction of an [`AccountBirthday`] from a [`TreeState`] +pub enum BirthdayError { + HeightInvalid(TryFromIntError), + Decode(io::Error), +} + +impl From for BirthdayError { + fn from(value: TryFromIntError) -> Self { + Self::HeightInvalid(value) + } +} + +impl From for BirthdayError { + fn from(value: io::Error) -> Self { + Self::Decode(value) + } } impl AccountBirthday { /// Constructs a new [`AccountBirthday`] from its constituent parts. /// - /// * `height`: The birthday height of the account. This is defined as the height of the last - /// block that is known to contain no transactions sent to addresses belonging to the account. + /// * `height`: The birthday height of the account. This is defined as the height of the first + /// block block to be scanned in wallet recovery. /// * `sapling_frontier`: The Sapling note commitment tree frontier as of the end of the block - /// at `height`. + /// prior to the birthday height. + /// * `recover_until`: An optional height at which the wallet should exit "recovery mode". In + /// order to avoid confusing shifts in wallet balance and spendability that may temporarily be + /// visible to a user during the process of recovering from seed, wallets may optionally set a + /// "recover until" height. The wallet is considered to be in "recovery mode" until there + /// exist no unscanned ranges between the wallet's birthday height and the provided + /// `recover_until` height, exclusive. + /// + /// This API is intended primarily to be used in testing contexts; under normal circumstances, + /// [`AccountBirthday::from_treestate`] should be used instead. + #[cfg(feature = "test-dependencies")] pub fn from_parts( height: BlockHeight, sapling_frontier: Frontier, + recover_until: Option, ) -> Self { Self { height, sapling_frontier, + recover_until, } } + /// Constructs a new [`AccountBirthday`] from a [`TreeState`] returned from `lightwalletd`. + /// + /// * `treestate`: The tree state corresponding to the last block prior to the wallet's + /// birthday height. + /// * `recover_until`: An optional height at which the wallet should exit "recovery mode". In + /// order to avoid confusing shifts in wallet balance and spendability that may temporarily be + /// visible to a user during the process of recovering from seed, wallets may optionally set a + /// "recover until" height. The wallet is considered to be in "recovery mode" until there + /// exist no unscanned ranges between the wallet's birthday height and the provided + /// `recover_until` height, exclusive. + pub fn from_treestate( + treestate: TreeState, + recover_until: Option, + ) -> Result { + Ok(Self { + height: BlockHeight::try_from(treestate.height + 1)?, + sapling_frontier: treestate.sapling_tree()?.to_frontier(), + recover_until, + }) + } + /// Returns the Sapling note commitment tree frontier as of the end of the block at /// [`Self::height`]. pub fn sapling_frontier(&self) -> &Frontier { @@ -501,6 +555,24 @@ impl AccountBirthday { pub fn height(&self) -> BlockHeight { self.height } + + /// Returns the height at which the wallet should exit "recovery mode". + pub fn recover_until(&self) -> Option { + self.recover_until + } + + #[cfg(feature = "test-dependencies")] + pub fn from_sapling_activation( + params: &P, + ) -> AccountBirthday { + use zcash_primitives::consensus::NetworkUpgrade; + + AccountBirthday::from_parts( + params.activation_height(NetworkUpgrade::Sapling).unwrap(), + Frontier::empty(), + None, + ) + } } /// This trait encapsulates the write capabilities required to update stored @@ -509,24 +581,36 @@ pub trait WalletWrite: WalletRead { /// The type of identifiers used to look up transparent UTXOs. type UtxoRef; - /// Tells the wallet to track the next available account-level spend authority, given - /// the current set of [ZIP 316] account identifiers known to the wallet database. + /// Tells the wallet to track the next available account-level spend authority, given the + /// current set of [ZIP 316] account identifiers known to the wallet database. + /// + /// Returns the account identifier for the newly-created wallet database entry, along with the + /// associated [`UnifiedSpendingKey`]. + /// + /// If a birthday height is having a height is below the current chain tip, this operation will + /// trigger a re-scan of the blocks at and above the provided height. The birthday height is + /// defined as the minimum block height that will be scanned for funds belonging to the wallet. /// - /// Returns the account identifier for the newly-created wallet database entry, along - /// with the associated [`UnifiedSpendingKey`]. + /// For new wallets, callers should construct the [`AccountBirthday`] using + /// [`AccountBirthday::from_treestate`] for the block at height `chain_tip_height - PRUNING_DEPTH`. + /// Setting the birthday height to a tree state below the pruning depth ensures that reorgs + /// cannot cause funds intended for the wallet to be missed; otherwise, if the chain tip height + /// were used for the wallet birthday, a transaction targeted at a height greater than the + /// chain tip could be mined at a height below that tip as part of a reorg. /// - /// If `seed` was imported from a backup and this method is being used to restore a - /// previous wallet state, you should use this method to add all of the desired - /// accounts before scanning the chain from the seed's birthday height. + /// If `seed` was imported from a backup and this method is being used to restore a previous + /// wallet state, you should use this method to add all of the desired accounts before scanning + /// the chain from the seed's birthday height. /// - /// By convention, wallets should only allow a new account to be generated after funds - /// have been received by the currently-available account (in order to enable - /// automated account recovery). + /// By convention, wallets should only allow a new account to be generated after funds have + /// been received by the currently-available account (in order to enable automated account + /// recovery). /// /// [ZIP 316]: https://zips.z.cash/zip-0316 fn create_account( &mut self, seed: &SecretVec, + birthday: AccountBirthday, ) -> Result<(AccountId, UnifiedSpendingKey), Self::Error>; /// Generates and persists the next available diversified address, given the current @@ -645,9 +729,9 @@ pub mod testing { }; use super::{ - chain::CommitmentTreeRoot, scanning::ScanRange, BlockMetadata, DecryptedTransaction, - NoteId, NullifierQuery, ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, - WalletWrite, SAPLING_SHARD_HEIGHT, + chain::CommitmentTreeRoot, scanning::ScanRange, AccountBirthday, BlockMetadata, + DecryptedTransaction, NoteId, NullifierQuery, ScannedBlock, SentTransaction, + WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }; pub struct MockWalletDb { @@ -818,6 +902,7 @@ pub mod testing { fn create_account( &mut self, seed: &SecretVec, + _birthday: AccountBirthday, ) -> Result<(AccountId, UnifiedSpendingKey), Self::Error> { let account = AccountId::from(0); UnifiedSpendingKey::from_seed(&self.network, seed.expose_secret(), account) diff --git a/zcash_client_backend/src/proto.rs b/zcash_client_backend/src/proto.rs index d6cc593eb9..711faface6 100644 --- a/zcash_client_backend/src/proto.rs +++ b/zcash_client_backend/src/proto.rs @@ -1,9 +1,13 @@ //! Generated code for handling light client protobuf structs. +use std::io; + +use incrementalmerkletree::frontier::CommitmentTree; use zcash_primitives::{ block::{BlockHash, BlockHeader}, consensus::BlockHeight, - sapling::{note::ExtractedNoteCommitment, Nullifier}, + merkle_tree::read_commitment_tree, + sapling::{note::ExtractedNoteCommitment, Node, Nullifier, NOTE_COMMITMENT_TREE_DEPTH}, transaction::{components::sapling, TxId}, }; @@ -141,3 +145,15 @@ impl compact_formats::CompactSaplingSpend { Nullifier::from_slice(&self.nf).map_err(|_| ()) } } + +impl service::TreeState { + pub fn sapling_tree(&self) -> io::Result> { + let sapling_tree_bytes = hex::decode(&self.sapling_tree).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Hex decoding of Sapling tree bytes failed: {:?}", e), + ) + })?; + read_commitment_tree::(&sapling_tree_bytes[..]) + } +} diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 02a9f57424..5037090d84 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -28,6 +28,7 @@ and this library adheres to Rust's notion of - `zcash_client_sqlite::error::SqliteClientError` has new error variants: - `SqliteClientError::BlockConflict` - `SqliteClientError::CacheMiss` + - `SqliteClientError::ChainHeightUnknown` - `zcash_client_backend::FsBlockDbError` has a new error variant: - `FsBlockDbError::CacheMiss` - `zcash_client_sqlite::FsBlockDb::write_block_metadata` now overwrites any @@ -38,6 +39,12 @@ and this library adheres to Rust's notion of - `zcash_client_sqlite::NoteId` has been replaced with `zcash_client_sqlite::ReceivedNoteId` as the `SentNoteId` variant of is now unused following changes to `zcash_client_backend::data_api::WalletRead`. +- `zcash_client_sqlite::wallet::init::{init_blocks_table, init_accounts_table}` + have been removed. `zcash_client_backend::data_api::WalletWrite::create_account` + should be used instead; the initialization of the note commitment tree + previously performed by `init_blocks_table` is now handled by passing an + `AccountBirthday` containing the note commitment tree frontier as of the + end of the birthday height block to `create_account` instead. ### Fixed - Fixed an off-by-one error in the `BlockSource` implementation for the SQLite-backed diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index c2b79467fd..a69bcce221 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -324,8 +324,6 @@ where mod tests { use std::num::NonZeroU32; - use secrecy::Secret; - use zcash_primitives::{ block::BlockHash, transaction::{components::Amount, fees::zip317::FeeRule}, @@ -335,8 +333,8 @@ mod tests { use zcash_client_backend::{ address::RecipientAddress, data_api::{ - chain::error::Error, wallet::input_selection::GreedyInputSelector, WalletRead, - WalletWrite, + chain::error::Error, wallet::input_selection::GreedyInputSelector, AccountBirthday, + WalletRead, }, fees::{zip317::SingleOutputChangeStrategy, DustOutputPolicy}, scanning::ScanError, @@ -345,7 +343,7 @@ mod tests { }; use crate::{ - testing::{birthday_at_sapling_activation, AddressType, TestBuilder}, + testing::{AddressType, TestBuilder}, wallet::{get_balance, truncate_to_height}, AccountId, }; @@ -354,7 +352,7 @@ mod tests { fn valid_chain_states() { let mut st = TestBuilder::new() .with_block_cache() - .with_test_account(birthday_at_sapling_activation) + .with_test_account(AccountBirthday::from_sapling_activation) .build(); let dfvk = st.test_account_sapling().unwrap(); @@ -387,7 +385,7 @@ mod tests { fn invalid_chain_cache_disconnected() { let mut st = TestBuilder::new() .with_block_cache() - .with_test_account(birthday_at_sapling_activation) + .with_test_account(AccountBirthday::from_sapling_activation) .build(); let dfvk = st.test_account_sapling().unwrap(); @@ -438,7 +436,7 @@ mod tests { fn data_db_truncation() { let mut st = TestBuilder::new() .with_block_cache() - .with_test_account(birthday_at_sapling_activation) + .with_test_account(AccountBirthday::from_sapling_activation) .build(); let dfvk = st.test_account_sapling().unwrap(); @@ -498,12 +496,13 @@ mod tests { #[test] fn scan_cached_blocks_allows_blocks_out_of_order() { - let mut st = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); - // Add an account to the wallet - let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); - let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); + let (_, usk, _) = st.test_account().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Create a block with height SAPLING_ACTIVATION_HEIGHT let value = Amount::from_u64(50000).unwrap(); @@ -558,7 +557,7 @@ mod tests { fn scan_cached_blocks_finds_received_notes() { let mut st = TestBuilder::new() .with_block_cache() - .with_test_account(birthday_at_sapling_activation) + .with_test_account(AccountBirthday::from_sapling_activation) .build(); let dfvk = st.test_account_sapling().unwrap(); @@ -600,9 +599,8 @@ mod tests { fn scan_cached_blocks_finds_change_notes() { let mut st = TestBuilder::new() .with_block_cache() - .with_test_account(birthday_at_sapling_activation) + .with_test_account(AccountBirthday::from_sapling_activation) .build(); - let dfvk = st.test_account_sapling().unwrap(); // Account balance should be zero @@ -645,7 +643,7 @@ mod tests { fn scan_cached_blocks_detects_spends_out_of_order() { let mut st = TestBuilder::new() .with_block_cache() - .with_test_account(birthday_at_sapling_activation) + .with_test_account(AccountBirthday::from_sapling_activation) .build(); let dfvk = st.test_account_sapling().unwrap(); diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 618b1d1c01..d7682aed20 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -86,7 +86,15 @@ pub enum SqliteClientError { /// commitment trees. CommitmentTree(ShardTreeError), + /// The block at the specified height was not available from the block cache. CacheMiss(BlockHeight), + + /// The height of the chain was not available; a call to [`WalletWrite::update_chain_tip`] is + /// required before the requested operation can succeed. + /// + /// [`WalletWrite::update_chain_tip`]: + /// zcash_client_backend::data_api::WalletWrite::update_chain_tip + ChainHeightUnknown, } impl error::Error for SqliteClientError { @@ -131,6 +139,7 @@ impl fmt::Display for SqliteClientError { SqliteClientError::AddressNotRecognized(_) => write!(f, "The address associated with a received txo is not identifiable as belonging to the wallet."), SqliteClientError::CommitmentTree(err) => write!(f, "An error occurred accessing or updating note commitment tree data: {}.", err), SqliteClientError::CacheMiss(height) => write!(f, "Requested height {} does not exist in the block cache.", height), + SqliteClientError::ChainHeightUnknown => write!(f, "Chain height unknown; please call `update_chain_tip`") } } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 93045c4c80..829abbbe63 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -64,9 +64,9 @@ use zcash_client_backend::{ self, chain::{BlockSource, CommitmentTreeRoot}, scanning::{ScanPriority, ScanRange}, - BlockMetadata, DecryptedTransaction, NoteId, NullifierQuery, PoolType, Recipient, - ScannedBlock, SentTransaction, ShieldedProtocol, WalletCommitmentTrees, WalletRead, - WalletWrite, SAPLING_SHARD_HEIGHT, + AccountBirthday, BlockMetadata, DecryptedTransaction, NoteId, NullifierQuery, PoolType, + Recipient, ScannedBlock, SentTransaction, ShieldedProtocol, WalletCommitmentTrees, + WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }, keys::{UnifiedFullViewingKey, UnifiedSpendingKey}, proto::compact_formats::CompactBlock, @@ -356,6 +356,7 @@ impl WalletWrite for WalletDb fn create_account( &mut self, seed: &SecretVec, + birthday: AccountBirthday, ) -> Result<(AccountId, UnifiedSpendingKey), Self::Error> { self.transactionally(|wdb| { let account = wallet::get_max_account_id(wdb.conn.0)? @@ -370,7 +371,7 @@ impl WalletWrite for WalletDb .map_err(|_| SqliteClientError::KeyDerivationError(account))?; let ufvk = usk.to_unified_full_viewing_key(); - wallet::add_account(wdb.conn.0, &wdb.params, account, &ufvk)?; + wallet::add_account(wdb.conn.0, &wdb.params, account, &ufvk, birthday)?; Ok((account, usk)) }) @@ -1085,26 +1086,21 @@ extern crate assert_matches; #[cfg(test)] mod tests { - use zcash_client_backend::data_api::{WalletRead, WalletWrite}; + use zcash_client_backend::data_api::{AccountBirthday, WalletRead, WalletWrite}; - use crate::{ - testing::{birthday_at_sapling_activation, TestBuilder}, - AccountId, - }; - - #[cfg(feature = "unstable")] - use zcash_primitives::{consensus::Parameters, transaction::components::Amount}; + use crate::{testing::TestBuilder, AccountId}; #[cfg(feature = "unstable")] - use zcash_client_backend::keys::sapling; - - #[cfg(feature = "unstable")] - use crate::testing::AddressType; + use { + crate::testing::AddressType, + zcash_client_backend::keys::sapling, + zcash_primitives::{consensus::Parameters, transaction::components::Amount}, + }; #[test] pub(crate) fn get_next_available_address() { let mut st = TestBuilder::new() - .with_test_account(birthday_at_sapling_activation) + .with_test_account(AccountBirthday::from_sapling_activation) .build(); let account = AccountId::from(0); @@ -1125,7 +1121,7 @@ mod tests { // Add an account to the wallet. let st = TestBuilder::new() .with_block_cache() - .with_test_account(birthday_at_sapling_activation) + .with_test_account(AccountBirthday::from_sapling_activation) .build(); let (_, usk, _) = st.test_account().unwrap(); diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index a309dcba95..d16af5a39a 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -130,7 +130,7 @@ impl TestBuilder { let test_account = if let Some(birthday) = self.test_account_birthday { let seed = Secret::new(vec![0u8; 32]); - let (account, usk) = db_data.create_account(&seed).unwrap(); + let (account, usk) = db_data.create_account(&seed, birthday.clone()).unwrap(); Some((account, usk, birthday)) } else { None @@ -532,17 +532,6 @@ impl TestState { } } -#[cfg(test)] -pub(crate) fn birthday_at_sapling_activation( - params: &P, -) -> AccountBirthday { - use incrementalmerkletree::frontier::Frontier; - AccountBirthday::from_parts( - params.activation_height(NetworkUpgrade::Sapling).unwrap(), - Frontier::empty(), - ) -} - #[allow(dead_code)] pub(crate) enum AddressType { DefaultExternal, diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 1b3f1a12a3..a2bf0433e9 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -64,15 +64,20 @@ //! wallet. //! - `memo` the shielded memo associated with the output, if any. +use incrementalmerkletree::Retention; use rusqlite::{self, named_params, OptionalExtension, ToSql}; +use shardtree::ShardTree; use std::cmp; use std::collections::HashMap; use std::convert::TryFrom; use std::io::{self, Cursor}; use std::num::NonZeroU32; -use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; +use tracing::debug; -use zcash_client_backend::data_api::{NoteId, ShieldedProtocol}; +use zcash_client_backend::data_api::{ + scanning::{ScanPriority, ScanRange}, + AccountBirthday, NoteId, ShieldedProtocol, SAPLING_SHARD_HEIGHT, +}; use zcash_primitives::transaction::TransactionData; use zcash_primitives::{ @@ -95,10 +100,11 @@ use zcash_client_backend::{ wallet::WalletTx, }; -use crate::VERIFY_LOOKAHEAD; +use crate::wallet::commitment_tree::SqliteShardStore; use crate::{ error::SqliteClientError, SqlTransaction, WalletCommitmentTrees, WalletDb, PRUNING_DEPTH, }; +use crate::{SAPLING_TABLES_PREFIX, VERIFY_LOOKAHEAD}; use self::scanning::replace_queue_entries; @@ -157,29 +163,86 @@ pub(crate) fn add_account( params: &P, account: AccountId, key: &UnifiedFullViewingKey, + birthday: AccountBirthday, ) -> Result<(), SqliteClientError> { - add_account_internal(conn, params, "accounts", account, key) -} + // Set the wallet birthday, falling back to the chain tip if not specified + let chain_tip = scan_queue_extrema(conn)?.map(|(_, max)| max); -pub(crate) fn add_account_internal>( - conn: &rusqlite::Transaction, - network: &P, - accounts_table: &'static str, - account: AccountId, - key: &UnifiedFullViewingKey, -) -> Result<(), E> { - let ufvk_str: String = key.encode(network); conn.execute( - &format!( - "INSERT INTO {} (account, ufvk) VALUES (:account, :ufvk)", - accounts_table - ), - named_params![":account": &::from(account), ":ufvk": &ufvk_str], + "INSERT INTO accounts (account, ufvk, birthday_height, recover_until_height) + VALUES (:account, :ufvk, :birthday_height, :recover_until_height)", + named_params![ + ":account": u32::from(account), + ":ufvk": &key.encode(params), + ":birthday_height": u32::from(birthday.height()), + ":recover_until_height": birthday.recover_until().map(u32::from) + ], )?; + // If a birthday frontier is available, insert it into the note commitment tree. If the + // birthday frontier is the empty frontier, we don't need to do anything. + if let Some(frontier) = birthday.sapling_frontier().value() { + debug!("Inserting frontier into ShardTree: {:?}", frontier); + let shard_store = SqliteShardStore::< + _, + zcash_primitives::sapling::Node, + SAPLING_SHARD_HEIGHT, + >::from_connection(conn, SAPLING_TABLES_PREFIX)?; + let mut shard_tree: ShardTree< + _, + { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + > = ShardTree::new(shard_store, PRUNING_DEPTH.try_into().unwrap()); + shard_tree.insert_frontier_nodes( + frontier.clone(), + Retention::Checkpoint { + // This subtraction is safe, because the non-empty frontier cannot exist at genesis + id: birthday.height() - 1, + is_marked: false, + }, + )?; + } + + let sapling_activation_height = params + .activation_height(NetworkUpgrade::Sapling) + .expect("Sapling activation height must be available."); + + // Add the ignored range up to and including the birthday height. + if sapling_activation_height < birthday.height() { + let ignored_range = sapling_activation_height..birthday.height(); + + replace_queue_entries::( + conn, + &ignored_range, + Some(ScanRange::from_parts( + ignored_range.clone(), + ScanPriority::Ignored, + )) + .into_iter(), + false, + )?; + }; + + // Rewrite the scan ranges above the birthday height so that we'll ensure we re-scan to find + // any notes that might belong to the newly added account. + if let Some(t) = chain_tip { + let rescan_range = birthday.height()..(t + 1); + + replace_queue_entries::( + conn, + &rescan_range, + Some(ScanRange::from_parts( + rescan_range.clone(), + ScanPriority::Historic, + )) + .into_iter(), + true, + )?; + } + // Always derive the default Unified Address for the account. let (address, d_idx) = key.default_address(); - insert_address(conn, network, account, d_idx, &address)?; + insert_address(conn, params, account, d_idx, &address)?; Ok(()) } @@ -235,17 +298,17 @@ pub(crate) fn insert_address( ) -> Result<(), rusqlite::Error> { let mut stmt = conn.prepare_cached( "INSERT INTO addresses ( - account, - diversifier_index_be, - address, - cached_transparent_receiver_address - ) - VALUES ( - :account, - :diversifier_index_be, - :address, - :cached_transparent_receiver_address - )", + account, + diversifier_index_be, + address, + cached_transparent_receiver_address + ) + VALUES ( + :account, + :diversifier_index_be, + :address, + :cached_transparent_receiver_address + )", )?; // the diversifier index is stored in big-endian order to allow sorting @@ -949,7 +1012,12 @@ pub(crate) fn truncate_to_height( // Prioritize the range starting at the height we just rewound to for verification let query_range = block_height..(block_height + VERIFY_LOOKAHEAD); let scan_range = ScanRange::from_parts(query_range.clone(), ScanPriority::Verify); - replace_queue_entries(conn, &query_range, Some(scan_range).into_iter(), false)?; + replace_queue_entries::( + conn, + &query_range, + Some(scan_range).into_iter(), + false, + )?; } Ok(()) @@ -1573,17 +1641,15 @@ mod tests { use zcash_primitives::transaction::components::Amount; - use zcash_client_backend::data_api::WalletRead; + use zcash_client_backend::data_api::{AccountBirthday, WalletRead}; - use crate::{ - testing::{birthday_at_sapling_activation, TestBuilder}, - AccountId, - }; + use crate::{testing::TestBuilder, AccountId}; use super::get_balance; #[cfg(feature = "transparent-inputs")] use { + incrementalmerkletree::frontier::Frontier, secrecy::Secret, zcash_client_backend::{ data_api::WalletWrite, encoding::AddressCodec, wallet::WalletTransparentOutput, @@ -1597,7 +1663,7 @@ mod tests { #[test] fn empty_database_has_no_balance() { let st = TestBuilder::new() - .with_test_account(birthday_at_sapling_activation) + .with_test_account(AccountBirthday::from_sapling_activation) .build(); // The account should be empty @@ -1628,11 +1694,15 @@ mod tests { #[test] #[cfg(feature = "transparent-inputs")] fn put_received_transparent_utxo() { + use crate::testing::TestBuilder; + let mut st = TestBuilder::new().build(); // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let (account_id, _usk) = st.wallet_mut().create_account(&seed).unwrap(); + let birthday = + AccountBirthday::from_parts(st.sapling_activation_height(), Frontier::empty(), None); + let (account_id, _usk) = st.wallet_mut().create_account(&seed, birthday).unwrap(); let uaddr = st .wallet() .get_current_address(account_id) diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 5856970aea..98162eebd6 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -1,39 +1,22 @@ //! Functions for initializing the various databases. -use incrementalmerkletree::Retention; -use std::{collections::HashMap, fmt}; -use tracing::debug; +use std::fmt; -use rusqlite::{self, named_params}; +use rusqlite::{self}; use schemer::{Migrator, MigratorError}; use schemer_rusqlite::RusqliteAdapter; use secrecy::SecretVec; -use shardtree::{error::ShardTreeError, ShardTree}; +use shardtree::error::ShardTreeError; use uuid::Uuid; use zcash_primitives::{ - block::BlockHash, - consensus::{self, BlockHeight, NetworkUpgrade}, - merkle_tree::read_commitment_tree, - sapling, + consensus::{self}, transaction::components::amount::BalanceError, - zip32::AccountId, }; -use zcash_client_backend::{ - data_api::{ - scanning::{ScanPriority, ScanRange}, - SAPLING_SHARD_HEIGHT, - }, - keys::UnifiedFullViewingKey, -}; - -use crate::{error::SqliteClientError, wallet, WalletDb, PRUNING_DEPTH, SAPLING_TABLES_PREFIX}; +use crate::WalletDb; -use super::{ - commitment_tree::{self, SqliteShardStore}, - scanning::insert_queue_entries, -}; +use super::commitment_tree::{self}; mod migrations; @@ -176,234 +159,37 @@ fn init_wallet_db_internal( Ok(()) } -/// Initialises the data database with the given set of account [`UnifiedFullViewingKey`]s. -/// -/// **WARNING** This method should be used with care, and should ordinarily be unnecessary. -/// Prefer to use [`WalletWrite::create_account`] instead. -/// -/// [`WalletWrite::create_account`]: zcash_client_backend::data_api::WalletWrite::create_account -/// -/// The [`UnifiedFullViewingKey`]s are stored internally and used by other APIs such as -/// [`scan_cached_blocks`], and [`create_spend_to_address`]. Account identifiers in `keys` **MUST** -/// form a consecutive sequence beginning at account 0, and the [`UnifiedFullViewingKey`] -/// corresponding to a given account identifier **MUST** be derived from the wallet's mnemonic seed -/// at the BIP-44 `account` path level as described by [ZIP -/// 316](https://zips.z.cash/zip-0316) -/// -/// # Examples -/// -/// ``` -/// # #[cfg(feature = "transparent-inputs")] -/// # { -/// use tempfile::NamedTempFile; -/// use secrecy::Secret; -/// use std::collections::HashMap; -/// -/// use zcash_primitives::{ -/// consensus::{Network, Parameters}, -/// zip32::{AccountId, ExtendedSpendingKey} -/// }; -/// -/// use zcash_client_backend::{ -/// keys::{ -/// sapling, -/// UnifiedFullViewingKey -/// }, -/// }; -/// -/// use zcash_client_sqlite::{ -/// WalletDb, -/// wallet::init::{init_accounts_table, init_wallet_db} -/// }; -/// -/// let data_file = NamedTempFile::new().unwrap(); -/// let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); -/// init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); -/// -/// let seed = [0u8; 32]; // insecure; replace with a strong random seed -/// let account = AccountId::from(0); -/// let extsk = sapling::spending_key(&seed, Network::TestNetwork.coin_type(), account); -/// let dfvk = extsk.to_diversifiable_full_viewing_key(); -/// let ufvk = UnifiedFullViewingKey::new(None, Some(dfvk), None).unwrap(); -/// let ufvks = HashMap::from([(account, ufvk)]); -/// init_accounts_table(&mut db_data, &ufvks).unwrap(); -/// # } -/// ``` -/// -/// [`get_address`]: crate::wallet::get_address -/// [`scan_cached_blocks`]: zcash_client_backend::data_api::chain::scan_cached_blocks -/// [`create_spend_to_address`]: zcash_client_backend::data_api::wallet::create_spend_to_address -pub fn init_accounts_table( - wallet_db: &mut WalletDb, - keys: &HashMap, -) -> Result<(), SqliteClientError> { - wallet_db.transactionally(|wdb| { - let mut empty_check = wdb.conn.0.prepare("SELECT * FROM accounts LIMIT 1")?; - if empty_check.exists([])? { - return Err(SqliteClientError::TableNotEmpty); - } - - // Ensure that the account identifiers are sequential and begin at zero. - if let Some(account_id) = keys.keys().max() { - if usize::try_from(u32::from(*account_id)).unwrap() >= keys.len() { - return Err(SqliteClientError::AccountIdDiscontinuity); - } - } - - // Insert accounts atomically - for (account, key) in keys.iter() { - wallet::add_account(wdb.conn.0, &wdb.params, *account, key)?; - } - - Ok(()) - }) -} - -/// Initialises the data database with the given block. -/// -/// This enables a newly-created database to be immediately-usable, without needing to -/// synchronise historic blocks. -/// -/// # Examples -/// -/// ``` -/// use tempfile::NamedTempFile; -/// use zcash_primitives::{ -/// block::BlockHash, -/// consensus::{BlockHeight, Network}, -/// }; -/// use zcash_client_sqlite::{ -/// WalletDb, -/// wallet::init::init_blocks_table, -/// }; -/// -/// // The block height. -/// let height = BlockHeight::from_u32(500_000); -/// // The hash of the block header. -/// let hash = BlockHash([0; 32]); -/// // The nTime field from the block header. -/// let time = 12_3456_7890; -/// // The serialized Sapling commitment tree as of this block. -/// // Pre-compute and hard-code, or obtain from a service. -/// let sapling_tree = &[]; -/// -/// let data_file = NamedTempFile::new().unwrap(); -/// let mut db = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); -/// init_blocks_table(&mut db, height, hash, time, sapling_tree); -/// ``` -pub fn init_blocks_table( - wallet_db: &mut WalletDb, - height: BlockHeight, - hash: BlockHash, - time: u32, - sapling_tree: &[u8], -) -> Result<(), SqliteClientError> { - wallet_db.transactionally(|wdb| { - let mut empty_check = wdb.conn.0.prepare("SELECT * FROM blocks LIMIT 1")?; - if empty_check.exists([])? { - return Err(SqliteClientError::TableNotEmpty); - } - - let block_end_tree = - read_commitment_tree::( - sapling_tree, - ) - .map_err(|e| { - rusqlite::Error::FromSqlConversionFailure( - sapling_tree.len(), - rusqlite::types::Type::Blob, - Box::new(e), - ) - })?; - - wdb.conn.0.execute( - "INSERT INTO blocks (height, hash, time, sapling_tree) - VALUES (:height, :hash, :time, :sapling_tree)", - named_params![ - ":height": u32::from(height), - ":hash": hash.0, - ":time": time, - ":sapling_tree": sapling_tree, - ], - )?; - - if let Some(sapling_activation) = wdb.params.activation_height(NetworkUpgrade::Sapling) { - let scan_range_start = std::cmp::min(sapling_activation, height); - let scan_range_end = height + 1; - debug!( - "Setting ignored block range {}..{}", - scan_range_start, scan_range_end - ); - insert_queue_entries( - wdb.conn.0, - Some(ScanRange::from_parts( - scan_range_start..scan_range_end, - ScanPriority::Ignored, - )) - .iter(), - )?; - } - - if let Some(nonempty_frontier) = block_end_tree.to_frontier().value() { - debug!("Inserting frontier into ShardTree: {:?}", nonempty_frontier); - let shard_store = - SqliteShardStore::<_, sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( - wdb.conn.0, - SAPLING_TABLES_PREFIX, - )?; - let mut shard_tree: ShardTree< - _, - { sapling::NOTE_COMMITMENT_TREE_DEPTH }, - SAPLING_SHARD_HEIGHT, - > = ShardTree::new(shard_store, PRUNING_DEPTH.try_into().unwrap()); - shard_tree.insert_frontier_nodes( - nonempty_frontier.clone(), - Retention::Checkpoint { - id: height, - is_marked: false, - }, - )?; - } - - Ok(()) - }) -} - #[cfg(test)] #[allow(deprecated)] mod tests { use rusqlite::{self, named_params, ToSql}; use secrecy::Secret; - use std::collections::HashMap; + use tempfile::NamedTempFile; use zcash_client_backend::{ address::RecipientAddress, - data_api::{scanning::ScanPriority, WalletRead}, + data_api::scanning::ScanPriority, encoding::{encode_extended_full_viewing_key, encode_payment_address}, keys::{sapling, UnifiedFullViewingKey, UnifiedSpendingKey}, }; use zcash_primitives::{ - block::BlockHash, consensus::{self, BlockHeight, BranchId, Network, NetworkUpgrade, Parameters}, transaction::{TransactionData, TxVersion}, - zip32::sapling::ExtendedFullViewingKey, + zip32::{sapling::ExtendedFullViewingKey, AccountId}, }; - use crate::{ - error::SqliteClientError, testing::TestBuilder, wallet::scanning::priority_code, AccountId, - WalletDb, - }; + use crate::{testing::TestBuilder, wallet::scanning::priority_code, WalletDb}; - use super::{init_accounts_table, init_blocks_table, init_wallet_db}; + use super::init_wallet_db; #[cfg(feature = "transparent-inputs")] use { crate::wallet::{self, pool_code, PoolType}, zcash_address::test_vectors, zcash_client_backend::data_api::WalletWrite, - zcash_primitives::{legacy::keys as transparent, zip32::DiversifierIndex}, + zcash_primitives::zip32::DiversifierIndex, }; #[test] @@ -416,8 +202,9 @@ mod tests { let expected_tables = vec![ "CREATE TABLE \"accounts\" ( account INTEGER PRIMARY KEY, - ufvk TEXT NOT NULL - )", + ufvk TEXT NOT NULL, + birthday_height INTEGER NOT NULL, + recover_until_height INTEGER )", "CREATE TABLE addresses ( account INTEGER NOT NULL, diversifier_index_be BLOB NOT NULL, @@ -581,6 +368,7 @@ mod tests { // v_sapling_shard_unscanned_ranges format!( "CREATE VIEW v_sapling_shard_unscanned_ranges AS + WITH wallet_birthday AS (SELECT MIN(birthday_height) AS height FROM accounts) SELECT shard.shard_index, shard.shard_index << 16 AS start_position, @@ -594,16 +382,18 @@ mod tests { FROM sapling_tree_shards shard LEFT OUTER JOIN sapling_tree_shards prev_shard ON shard.shard_index = prev_shard.shard_index + 1 - INNER JOIN scan_queue ON + INNER JOIN scan_queue ON (scan_queue.block_range_start BETWEEN subtree_start_height AND shard.subtree_end_height) OR ((scan_queue.block_range_end - 1) BETWEEN subtree_start_height AND shard.subtree_end_height) OR ( scan_queue.block_range_start <= prev_shard.subtree_end_height AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height ) - WHERE scan_queue.priority > {}", + INNER JOIN wallet_birthday + WHERE scan_queue.priority > {} + AND scan_queue.block_range_end > wallet_birthday.height", u32::from(st.network().activation_height(NetworkUpgrade::Sapling).unwrap()), - priority_code(&ScanPriority::Scanned), + priority_code(&ScanPriority::Scanned) ), // v_transactions "CREATE VIEW v_transactions AS @@ -875,7 +665,10 @@ mod tests { let extfvk = secret_key.to_extended_full_viewing_key(); init_0_3_0(&mut db_data, &extfvk, account).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(seed.to_vec()))).unwrap(); + assert_matches!( + init_wallet_db(&mut db_data, Some(Secret::new(seed.to_vec()))), + Ok(_) + ); } #[test] @@ -1043,7 +836,10 @@ mod tests { let extfvk = secret_key.to_extended_full_viewing_key(); init_autoshielding(&mut db_data, &extfvk, account).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(seed.to_vec()))).unwrap(); + assert_matches!( + init_wallet_db(&mut db_data, Some(Secret::new(seed.to_vec()))), + Ok(_) + ); } #[test] @@ -1197,136 +993,30 @@ mod tests { account, ) .unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(seed.to_vec()))).unwrap(); - } - - #[test] - fn init_accounts_table_only_works_once() { - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); - - // We can call the function as many times as we want with no data - init_accounts_table(&mut db_data, &HashMap::new()).unwrap(); - init_accounts_table(&mut db_data, &HashMap::new()).unwrap(); - - let seed = [0u8; 32]; - let account = AccountId::from(0); - - // First call with data should initialise the accounts table - let extsk = sapling::spending_key(&seed, db_data.params.coin_type(), account); - let dfvk = extsk.to_diversifiable_full_viewing_key(); - - #[cfg(feature = "transparent-inputs")] - let ufvk = UnifiedFullViewingKey::new( - Some( - transparent::AccountPrivKey::from_seed(&db_data.params, &seed, account) - .unwrap() - .to_account_pubkey(), - ), - Some(dfvk), - None, - ) - .unwrap(); - - #[cfg(not(feature = "transparent-inputs"))] - let ufvk = UnifiedFullViewingKey::new(Some(dfvk), None).unwrap(); - let ufvks = HashMap::from([(account, ufvk)]); - - init_accounts_table(&mut db_data, &ufvks).unwrap(); - - // Subsequent calls should return an error - init_accounts_table(&mut db_data, &HashMap::new()).unwrap_err(); - init_accounts_table(&mut db_data, &ufvks).unwrap_err(); - } - - #[test] - fn init_accounts_table_allows_no_gaps() { - let params = Network::TestNetwork; - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), params).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); - - // allow sequential initialization - let seed = [0u8; 32]; - let ufvks = |ids: &[u32]| { - ids.iter() - .map(|a| { - let account = AccountId::from(*a); - UnifiedSpendingKey::from_seed(¶ms, &seed, account) - .map(|k| (account, k.to_unified_full_viewing_key())) - .unwrap() - }) - .collect::>() - }; - - // should fail if we have a gap assert_matches!( - init_accounts_table(&mut db_data, &ufvks(&[0, 2])), - Err(SqliteClientError::AccountIdDiscontinuity) + init_wallet_db(&mut db_data, Some(Secret::new(seed.to_vec()))), + Ok(_) ); - - // should succeed if there are no gaps - assert!(init_accounts_table(&mut db_data, &ufvks(&[0, 1, 2])).is_ok()); - } - - #[test] - fn init_blocks_table_only_works_once() { - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); - init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); - - // First call with data should initialise the blocks table - init_blocks_table( - &mut db_data, - BlockHeight::from(1u32), - BlockHash([1; 32]), - 1, - &[0x0, 0x0, 0x0], - ) - .unwrap(); - - // Subsequent calls should return an error - init_blocks_table( - &mut db_data, - BlockHeight::from(2u32), - BlockHash([2; 32]), - 2, - &[0x0, 0x0, 0x0], - ) - .unwrap_err(); - } - - #[test] - fn init_accounts_table_stores_correct_address() { - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); - init_wallet_db(&mut db_data, None).unwrap(); - - let seed = [0u8; 32]; - - // Add an account to the wallet - let account_id = AccountId::from(0); - let usk = UnifiedSpendingKey::from_seed(&db_data.params, &seed, account_id).unwrap(); - let ufvk = usk.to_unified_full_viewing_key(); - let expected_address = ufvk.sapling().unwrap().default_address().1; - let ufvks = HashMap::from([(account_id, ufvk)]); - init_accounts_table(&mut db_data, &ufvks).unwrap(); - - // The account's address should be in the data DB - let ua = db_data.get_current_address(AccountId::from(0)).unwrap(); - assert_eq!(ua.unwrap().sapling().unwrap(), &expected_address); } #[test] #[cfg(feature = "transparent-inputs")] fn account_produces_expected_ua_sequence() { - let data_file = NamedTempFile::new().unwrap(); - let mut db_data = WalletDb::for_path(data_file.path(), Network::MainNetwork).unwrap(); - init_wallet_db(&mut db_data, None).unwrap(); + use zcash_client_backend::data_api::AccountBirthday; + let network = Network::MainNetwork; + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), network).unwrap(); let seed = test_vectors::UNIFIED[0].root_seed; - let (account, _usk) = db_data.create_account(&Secret::new(seed.to_vec())).unwrap(); + assert_matches!( + init_wallet_db(&mut db_data, Some(Secret::new(seed.to_vec()))), + Ok(_) + ); + + let birthday = AccountBirthday::from_sapling_activation(&network); + let (account, _usk) = db_data + .create_account(&Secret::new(seed.to_vec()), birthday) + .unwrap(); assert_eq!(account, AccountId::from(0u32)); for tv in &test_vectors::UNIFIED[..3] { diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 6fbaea7e20..f5cd2e2ac2 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -1,3 +1,4 @@ +mod add_account_birthdays; mod add_transaction_views; mod add_utxo_account; mod addresses_table; @@ -38,6 +39,8 @@ pub(super) fn all_migrations( // / | \ // shardtree_support nullifier_map sapling_memo_consistency // | + // add_account_birthdays + // | // v_sapling_shard_unscanned_ranges vec![ Box::new(initial_setup::Migration {}), @@ -63,6 +66,9 @@ pub(super) fn all_migrations( Box::new(sapling_memo_consistency::Migration { params: params.clone(), }), + Box::new(add_account_birthdays::Migration { + params: params.clone(), + }), Box::new(v_sapling_shard_unscanned_ranges::Migration { params: params.clone(), }), diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_account_birthdays.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_account_birthdays.rs new file mode 100644 index 0000000000..bf1e5e380d --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_account_birthdays.rs @@ -0,0 +1,75 @@ +//! This migration adds a birthday height to each account record. + +use std::collections::HashSet; + +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; +use zcash_primitives::consensus::{self, NetworkUpgrade}; + +use crate::wallet::init::WalletMigrationError; + +use super::shardtree_support; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xeeec0d0d_fee0_4231_8c68_5f3a7c7c2245); + +pub(super) struct Migration

{ + pub(super) params: P, +} + +impl

schemer::Migration for Migration

{ + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [shardtree_support::MIGRATION_ID].into_iter().collect() + } + + fn description(&self) -> &'static str { + "Adds a birthday height for each account." + } +} + +impl RusqliteMigration for Migration

{ + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + transaction.execute_batch(&format!( + "ALTER TABLE accounts ADD COLUMN birthday_height INTEGER; + ALTER TABLE accounts ADD COLUMN recover_until_height INTEGER; + + -- set the birthday height to the height of the first block in the blocks table + UPDATE accounts SET birthday_height = MIN(blocks.height) FROM blocks; + -- if the blocks table is empty, set the birthday height to Sapling activation - 1 + UPDATE accounts SET birthday_height = {} WHERE birthday_height IS NULL; + + CREATE TABLE accounts_new ( + account INTEGER PRIMARY KEY, + ufvk TEXT NOT NULL, + birthday_height INTEGER NOT NULL, + recover_until_height INTEGER + ); + + INSERT INTO accounts_new (account, ufvk, birthday_height) + SELECT account, ufvk, birthday_height FROM accounts; + + PRAGMA foreign_keys=OFF; + PRAGMA legacy_alter_table = ON; + DROP TABLE accounts; + ALTER TABLE accounts_new RENAME TO accounts; + PRAGMA legacy_alter_table = OFF; + PRAGMA foreign_keys=ON;", + u32::from( + self.params + .activation_height(NetworkUpgrade::Sapling) + .unwrap() + ) + ))?; + + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + panic!("This migration cannot be reverted."); + } +} diff --git a/zcash_client_sqlite/src/wallet/init/migrations/addresses_table.rs b/zcash_client_sqlite/src/wallet/init/migrations/addresses_table.rs index 3d3a7fa95d..7601f8d387 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/addresses_table.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/addresses_table.rs @@ -1,13 +1,13 @@ use std::collections::HashSet; -use rusqlite::Transaction; +use rusqlite::{named_params, Transaction}; use schemer; use schemer_rusqlite::RusqliteMigration; use uuid::Uuid; use zcash_client_backend::{address::RecipientAddress, keys::UnifiedFullViewingKey}; use zcash_primitives::{consensus, zip32::AccountId}; -use crate::wallet::{add_account_internal, init::WalletMigrationError}; +use crate::wallet::{init::WalletMigrationError, insert_address}; #[cfg(feature = "transparent-inputs")] use zcash_primitives::legacy::keys::IncomingViewingKey; @@ -152,13 +152,17 @@ impl RusqliteMigration for Migration

{ } } - add_account_internal::( - transaction, - &self.params, - "accounts_new", - account, - &ufvk, + transaction.execute( + "INSERT INTO accounts_new (account, ufvk) + VALUES (:account, :ufvk)", + named_params![ + ":account": u32::from(account), + ":ufvk": ufvk.encode(&self.params), + ], )?; + + let (address, d_idx) = ufvk.default_address(); + insert_address(transaction, &self.params, account, d_idx, &address)?; } transaction.execute_batch( diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs index 770ecddc5e..ea1aa68ce2 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs @@ -6,11 +6,11 @@ use std::collections::HashSet; use schemer_rusqlite::RusqliteMigration; use uuid::Uuid; use zcash_client_backend::data_api::{scanning::ScanPriority, SAPLING_SHARD_HEIGHT}; -use zcash_primitives::consensus; +use zcash_primitives::consensus::{self, NetworkUpgrade}; use crate::wallet::{init::WalletMigrationError, scanning::priority_code}; -use super::shardtree_support; +use super::add_account_birthdays; pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xfa934bdc_97b6_4980_8a83_b2cb1ac465fd); @@ -24,7 +24,7 @@ impl

schemer::Migration for Migration

{ } fn dependencies(&self) -> HashSet { - [shardtree_support::MIGRATION_ID].into_iter().collect() + [add_account_birthdays::MIGRATION_ID].into_iter().collect() } fn description(&self) -> &'static str { @@ -39,6 +39,7 @@ impl RusqliteMigration for Migration

{ transaction.execute_batch( &format!( "CREATE VIEW v_sapling_shard_unscanned_ranges AS + WITH wallet_birthday AS (SELECT MIN(birthday_height) AS height FROM accounts) SELECT shard.shard_index, shard.shard_index << {} AS start_position, @@ -52,17 +53,19 @@ impl RusqliteMigration for Migration

{ FROM sapling_tree_shards shard LEFT OUTER JOIN sapling_tree_shards prev_shard ON shard.shard_index = prev_shard.shard_index + 1 - INNER JOIN scan_queue ON + INNER JOIN scan_queue ON (scan_queue.block_range_start BETWEEN subtree_start_height AND shard.subtree_end_height) OR ((scan_queue.block_range_end - 1) BETWEEN subtree_start_height AND shard.subtree_end_height) OR ( scan_queue.block_range_start <= prev_shard.subtree_end_height AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height ) - WHERE scan_queue.priority > {}", + INNER JOIN wallet_birthday + WHERE scan_queue.priority > {} + AND scan_queue.block_range_end > wallet_birthday.height;", SAPLING_SHARD_HEIGHT, SAPLING_SHARD_HEIGHT, - u32::from(self.params.activation_height(consensus::NetworkUpgrade::Sapling).unwrap()), + u32::from(self.params.activation_height(NetworkUpgrade::Sapling).unwrap()), priority_code(&ScanPriority::Scanned), ) )?; diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 9c254c7890..564a785c49 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -403,8 +403,6 @@ pub(crate) fn put_received_note( pub(crate) mod tests { use std::{convert::Infallible, num::NonZeroU32}; - use secrecy::Secret; - use zcash_proofs::prover::LocalTxProver; use zcash_primitives::{ @@ -428,7 +426,7 @@ pub(crate) mod tests { self, error::Error, wallet::input_selection::{GreedyInputSelector, GreedyInputSelectorError}, - ShieldedProtocol, WalletRead, WalletWrite, + AccountBirthday, ShieldedProtocol, WalletRead, }, decrypt_transaction, fees::{fixed, zip317, DustOutputPolicy}, @@ -446,7 +444,7 @@ pub(crate) mod tests { #[cfg(feature = "transparent-inputs")] use { - zcash_client_backend::wallet::WalletTransparentOutput, + zcash_client_backend::{data_api::WalletWrite, wallet::WalletTransparentOutput}, zcash_primitives::transaction::components::{amount::NonNegativeAmount, OutPoint, TxOut}, }; @@ -461,12 +459,13 @@ pub(crate) mod tests { #[test] fn send_proposed_transfer() { - let mut st = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); - // Add an account to the wallet - let seed = Secret::new([0u8; 32].to_vec()); - let (account, usk) = st.wallet_mut().create_account(&seed).unwrap(); - let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); + let (account, usk, _) = st.test_account().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note let value = Amount::from_u64(60000).unwrap(); @@ -609,12 +608,10 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_on_incorrect_usk() { - let mut st = TestBuilder::new().build(); - - // Add an account to the wallet - let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); - let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); + let mut st = TestBuilder::new() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + let dfvk = st.test_account_sapling().unwrap(); let to = dfvk.default_address().1.into(); // Create a USK that doesn't exist in the wallet @@ -637,12 +634,12 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_with_no_blocks() { - let mut st = TestBuilder::new().build(); + let mut st = TestBuilder::new() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); - // Add an account to the wallet - let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); - let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); + let (_, usk, _) = st.test_account().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); let to = dfvk.default_address().1.into(); // Account balance should be zero @@ -667,12 +664,13 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_on_unverified_notes() { - let mut st = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); - // Add an account to the wallet - let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); - let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); + let (_, usk, _) = st.test_account().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); @@ -778,12 +776,13 @@ pub(crate) mod tests { #[test] fn create_to_address_fails_on_locked_notes() { - let mut st = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); - // Add an account to the wallet - let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); - let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); + let (_, usk, _) = st.test_account().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); @@ -876,12 +875,13 @@ pub(crate) mod tests { #[test] fn ovk_policy_prevents_recovery_from_chain() { - let mut st = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); - // Add an account to the wallet - let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); - let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); + let (_, usk, _) = st.test_account().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); @@ -976,12 +976,13 @@ pub(crate) mod tests { #[test] fn create_to_address_succeeds_to_t_addr_zero_change() { - let mut st = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); - // Add an account to the wallet - let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); - let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); + let (_, usk, _) = st.test_account().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note let value = Amount::from_u64(60000).unwrap(); @@ -1019,12 +1020,13 @@ pub(crate) mod tests { #[test] fn create_to_address_spends_a_change_note() { - let mut st = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); - // Add an account to the wallet - let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); - let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); + let (_, usk, _) = st.test_account().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note let value = Amount::from_u64(60000).unwrap(); @@ -1062,12 +1064,13 @@ pub(crate) mod tests { #[test] fn zip317_spend() { - let mut st = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); - // Add an account to the wallet - let seed = Secret::new([0u8; 32].to_vec()); - let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); - let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); + let (_, usk, _) = st.test_account().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet let (h1, _, _) = st.generate_next_block( @@ -1159,12 +1162,14 @@ pub(crate) mod tests { #[test] #[cfg(feature = "transparent-inputs")] fn shield_transparent() { - let mut st = TestBuilder::new().with_block_cache().build(); + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account_id, usk, _) = st.test_account().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); - // Add an account to the wallet - let seed = Secret::new([0u8; 32].to_vec()); - let (account_id, usk) = st.wallet_mut().create_account(&seed).unwrap(); - let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); let uaddr = st .wallet() .get_current_address(account_id) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 192d7b457f..69030a15a5 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -1,20 +1,22 @@ use rusqlite::{self, named_params, types::Value, OptionalExtension}; +use shardtree::error::ShardTreeError; use std::cmp::{max, min, Ordering}; use std::collections::BTreeSet; use std::ops::{Not, Range}; use std::rc::Rc; use tracing::{debug, trace}; -use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; use incrementalmerkletree::{Address, Position}; +use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; use zcash_primitives::consensus::{self, BlockHeight, NetworkUpgrade}; use zcash_client_backend::data_api::SAPLING_SHARD_HEIGHT; -use crate::error::SqliteClientError; -use crate::{PRUNING_DEPTH, VERIFY_LOOKAHEAD}; - -use super::block_height_extrema; +use crate::{ + error::SqliteClientError, + wallet::{block_height_extrema, commitment_tree, init::WalletMigrationError}, + PRUNING_DEPTH, VERIFY_LOOKAHEAD, +}; #[derive(Debug, Clone, Copy)] enum InsertOn { @@ -481,15 +483,66 @@ pub(crate) fn insert_queue_entries<'a>( Ok(()) } -pub(crate) fn replace_queue_entries( +/// A trait that abstracts over the construction of wallet errors. +/// +/// In order to make it possible to use [`replace_queue_entries`] in database migrations as well as +/// in code that returns `SqliteClientError`, it is necessary for that method to be polymorphic in +/// the error type. +pub(crate) trait WalletError { + fn db_error(err: rusqlite::Error) -> Self; + fn corrupt(message: String) -> Self; + fn chain_height_unknown() -> Self; + fn commitment_tree(err: ShardTreeError) -> Self; +} + +impl WalletError for SqliteClientError { + fn db_error(err: rusqlite::Error) -> Self { + SqliteClientError::DbError(err) + } + + fn corrupt(message: String) -> Self { + SqliteClientError::CorruptedData(message) + } + + fn chain_height_unknown() -> Self { + SqliteClientError::ChainHeightUnknown + } + + fn commitment_tree(err: ShardTreeError) -> Self { + SqliteClientError::CommitmentTree(err) + } +} + +impl WalletError for WalletMigrationError { + fn db_error(err: rusqlite::Error) -> Self { + WalletMigrationError::DbError(err) + } + + fn corrupt(message: String) -> Self { + WalletMigrationError::CorruptedData(message) + } + + fn chain_height_unknown() -> Self { + WalletMigrationError::CorruptedData( + "Wallet migration requires a valid account birthday.".to_owned(), + ) + } + + fn commitment_tree(err: ShardTreeError) -> Self { + WalletMigrationError::CommitmentTree(err) + } +} + +pub(crate) fn replace_queue_entries( conn: &rusqlite::Transaction<'_>, query_range: &Range, entries: impl Iterator, force_rescans: bool, -) -> Result<(), SqliteClientError> { +) -> Result<(), E> { let (to_create, to_delete_ends) = { - let mut suggested_stmt = conn.prepare_cached( - "SELECT block_range_start, block_range_end, priority + let mut suggested_stmt = conn + .prepare_cached( + "SELECT block_range_start, block_range_end, priority FROM scan_queue WHERE ( -- the start is contained within or adjacent to the range @@ -507,12 +560,15 @@ pub(crate) fn replace_queue_entries( AND block_range_end <= :end ) ORDER BY block_range_end", - )?; + ) + .map_err(E::db_error)?; - let mut rows = suggested_stmt.query(named_params![ - ":start": u32::from(query_range.start), - ":end": u32::from(query_range.end), - ])?; + let mut rows = suggested_stmt + .query(named_params![ + ":start": u32::from(query_range.start), + ":end": u32::from(query_range.end), + ]) + .map_err(E::db_error)?; // Iterate over the ranges in the scan queue that overlap the range that we have // identified as needing to be fully scanned. For each such range add it to the @@ -520,19 +576,16 @@ pub(crate) fn replace_queue_entries( // some in the process). let mut to_create: Option = None; let mut to_delete_ends: Vec = vec![]; - while let Some(row) = rows.next()? { + while let Some(row) = rows.next().map_err(E::db_error)? { let entry = ScanRange::from_parts( Range { - start: BlockHeight::from(row.get::<_, u32>(0)?), - end: BlockHeight::from(row.get::<_, u32>(1)?), + start: BlockHeight::from(row.get::<_, u32>(0).map_err(E::db_error)?), + end: BlockHeight::from(row.get::<_, u32>(1).map_err(E::db_error)?), }, { - let code = row.get::<_, i64>(2)?; + let code = row.get::<_, i64>(2).map_err(E::db_error)?; parse_priority_code(code).ok_or_else(|| { - SqliteClientError::CorruptedData(format!( - "scan priority not recognized: {}", - code - )) + E::corrupt(format!("scan priority not recognized: {}", code)) })? }, ); @@ -562,10 +615,11 @@ pub(crate) fn replace_queue_entries( conn.execute( "DELETE FROM scan_queue WHERE block_range_end IN rarray(:ends)", named_params![":ends": ends_ptr], - )?; + ) + .map_err(E::db_error)?; let scan_ranges = tree.into_vec(); - insert_queue_entries(conn, scan_ranges.iter())?; + insert_queue_entries(conn, scan_ranges.iter()).map_err(E::db_error)?; } Ok(()) @@ -646,7 +700,7 @@ pub(crate) fn scan_complete( vec![] }; - replace_queue_entries( + replace_queue_entries::( conn, &query_range, Some(scanned).into_iter().chain(extensions.into_iter()), @@ -750,7 +804,7 @@ pub(crate) fn update_chain_tip( }; if let Some(query_range) = query_range { - replace_queue_entries( + replace_queue_entries::( conn, &query_range, shard_entry.into_iter().chain(tip_entry.into_iter()), @@ -778,13 +832,12 @@ pub(crate) fn update_chain_tip( mod tests { use std::ops::Range; - use incrementalmerkletree::{Hashable, Level}; - use secrecy::Secret; + use incrementalmerkletree::{frontier::Frontier, Hashable, Level, Position}; use zcash_client_backend::data_api::{ chain::CommitmentTreeRoot, scanning::{ScanPriority, ScanRange}, - WalletCommitmentTrees, WalletRead, WalletWrite, + AccountBirthday, WalletCommitmentTrees, WalletRead, WalletWrite, }; use zcash_primitives::{ block::BlockHash, @@ -794,11 +847,9 @@ mod tests { }; use crate::{ - testing::{birthday_at_sapling_activation, AddressType, TestBuilder}, - wallet::{ - init::init_blocks_table, - scanning::{insert_queue_entries, replace_queue_entries, suggest_scan_ranges}, - }, + error::SqliteClientError, + testing::{AddressType, TestBuilder}, + wallet::scanning::{insert_queue_entries, replace_queue_entries, suggest_scan_ranges}, }; use super::{join_nonoverlapping, Joined, RangeOrdering, SpanningTree}; @@ -1160,7 +1211,7 @@ mod tests { let mut st = TestBuilder::new() .with_block_cache() - .with_test_account(birthday_at_sapling_activation) + .with_test_account(AccountBirthday::from_sapling_activation) .build(); let dfvk = st.test_account_sapling().unwrap(); @@ -1274,42 +1325,33 @@ mod tests { fn create_account_creates_ignored_range() { use ScanPriority::*; - let mut st = TestBuilder::new().build(); + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(|network| { + // We use Canopy activation as an arbitrary birthday height that's greater than Sapling + // activation. + let birthday_height = network.activation_height(NetworkUpgrade::Canopy).unwrap(); + let frontier_position = Position::from((0x1 << 16) + 1234); + let frontier = Frontier::from_parts( + frontier_position, + Node::empty_leaf(), + vec![Node::empty_leaf(); frontier_position.past_ommer_count().into()], + ) + .unwrap(); + AccountBirthday::from_parts(birthday_height, frontier, None) + }) + .build(); + let (_, _, birthday) = st.test_account().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); let sap_active = st.sapling_activation_height(); - // We use Canopy activation as an arbitrary birthday height that's greater than Sapling - // activation. - let birthday_height = st - .network() - .activation_height(NetworkUpgrade::Canopy) - .unwrap(); - - // call `init_blocks_table` to initialize the scan queue - init_blocks_table( - st.wallet_mut(), - birthday_height, - BlockHash([1; 32]), - 1, - &[0x0, 0x0, 0x0], - ) - .unwrap(); - - let seed = Secret::new(vec![0u8; 32]); - let (_, usk) = st.wallet_mut().create_account(&seed).unwrap(); - let _dfvk = usk.to_unified_full_viewing_key().sapling().unwrap().clone(); - let expected = vec![ // The range up to and including the wallet's birthday height is ignored. - scan_range( - u32::from(sap_active)..u32::from(birthday_height + 1), - Ignored, - ), + scan_range(u32::from(sap_active)..u32::from(birthday.height()), Ignored), ]; - assert_matches!( - suggest_scan_ranges(&st.wallet().conn, Ignored), - Ok(scan_ranges) if scan_ranges == expected - ); + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, expected); // Set up some shard history st.wallet_mut() @@ -1322,7 +1364,7 @@ mod tests { // complete the left-hand side of the required shard; this can be fixed once we // have proper account birthdays. CommitmentTreeRoot::from_parts( - birthday_height - 1000, + birthday.height() - 1000, // fake a hash, the value doesn't matter Node::empty_leaf(), ), @@ -1330,6 +1372,18 @@ mod tests { ) .unwrap(); + st.generate_block_at( + birthday.height(), + BlockHash([0u8; 32]), + &dfvk, + AddressType::DefaultExternal, + Amount::const_from_i64(10000), + u64::from(birthday.sapling_frontier().value().unwrap().position() + 1) + .try_into() + .unwrap(), + ); + st.scan_cached_blocks(birthday.height(), 1); + // Update the chain tip let tip_height = st .wallet() @@ -1343,25 +1397,25 @@ mod tests { // The birthday height was "last scanned" (as the wallet birthday) so we verify 10 // blocks starting at that height. scan_range( - u32::from(birthday_height)..u32::from(birthday_height + 10), + u32::from(birthday.height())..u32::from(birthday.height() + 10), Verify, ), // The remainder of the shard after the verify segment is required in order to make // notes spendable, so it has priority `ChainTip` scan_range( - u32::from(birthday_height + 10)..u32::from(tip_height + 1), + u32::from(birthday.height() + 10)..u32::from(tip_height + 1), ChainTip, ), // The remainder of the shard prior to the birthday height must be scanned because the // wallet doesn't know that it already has enough data from the initial frontier to // avoid having to scan this range. scan_range( - u32::from(birthday_height - 1000)..u32::from(birthday_height), + u32::from(birthday.height() - 1000)..u32::from(birthday.height()), ChainTip, ), // The range below the wallet's birthday height is ignored scan_range( - u32::from(sap_active)..u32::from(birthday_height - 1000), + u32::from(sap_active)..u32::from(birthday.height() - 1000), Ignored, ), ]; @@ -1393,7 +1447,7 @@ mod tests { { let tx = st.wallet_mut().conn.transaction().unwrap(); - replace_queue_entries( + replace_queue_entries::( &tx, &(BlockHeight::from(150)..BlockHeight::from(160)), vec![scan_range(150..160, Scanned)].into_iter(), @@ -1436,7 +1490,7 @@ mod tests { { let tx = st.wallet_mut().conn.transaction().unwrap(); - replace_queue_entries( + replace_queue_entries::( &tx, &(BlockHeight::from(90)..BlockHeight::from(100)), vec![scan_range(90..100, Scanned)].into_iter(), From 0b0274cdc127fbc85034e16392c6d7f78d2d578e Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 16 Aug 2023 11:15:10 -0600 Subject: [PATCH 0129/1122] zcash_client_backend: Make wallet and account birthday heights available via the data access API --- zcash_client_backend/CHANGELOG.md | 3 ++- zcash_client_backend/src/data_api.rs | 18 +++++++++++++++ zcash_client_sqlite/CHANGELOG.md | 3 ++- zcash_client_sqlite/src/error.rs | 5 ++++ zcash_client_sqlite/src/lib.rs | 8 +++++++ zcash_client_sqlite/src/wallet.rs | 34 ++++++++++++++++++++++++++++ 6 files changed, 69 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index b06c540af7..80042ceade 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -29,7 +29,8 @@ and this library adheres to Rust's notion of - `ScannedBlock` - `ShieldedProtocol` - `WalletCommitmentTrees` - - `WalletRead::{chain_height, block_metadata, block_fully_scanned, suggest_scan_ranges}` + - `WalletRead::{chain_height, block_metadata, block_fully_scanned, suggest_scan_ranges, + get_wallet_birthday, get_account_birthday}` - `WalletWrite::{put_blocks, update_chain_tip}` - `chain::CommitmentTreeRoot` - `scanning` A new module containing types required for `suggest_scan_ranges` diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 9909ace188..3ef4eb2afc 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -119,6 +119,16 @@ pub trait WalletRead { /// transaction is not in the main chain. fn get_tx_height(&self, txid: TxId) -> Result, Self::Error>; + /// Returns the birthday height for the wallet. + /// + /// This returns earliest birthday height among accounts maintained by this wallet, or + /// `Ok(None)` if the wallet has no initialized accounts. + fn get_wallet_birthday(&self) -> Result, Self::Error>; + + /// Returns the birthday height for the given account, or an error if the account is not known + /// to the wallet. + fn get_account_birthday(&self, account: AccountId) -> Result; + /// Returns the most recently generated unified address for the specified account, if the /// account identifier specified refers to a valid account for this wallet. /// @@ -801,6 +811,14 @@ pub mod testing { Ok(None) } + fn get_wallet_birthday(&self) -> Result, Self::Error> { + Ok(None) + } + + fn get_account_birthday(&self, _account: AccountId) -> Result { + Err(()) + } + fn get_current_address( &self, _account: AccountId, diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 5037090d84..33b7e128e0 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -26,6 +26,7 @@ and this library adheres to Rust's notion of wallet did not contain enough observed blocks to satisfy the `min_confirmations` value specified; this situation is now treated as an error. - `zcash_client_sqlite::error::SqliteClientError` has new error variants: + - `SqliteClientError::AccountUnknown` - `SqliteClientError::BlockConflict` - `SqliteClientError::CacheMiss` - `SqliteClientError::ChainHeightUnknown` @@ -37,7 +38,7 @@ and this library adheres to Rust's notion of ### Removed - The empty `wallet::transact` module has been removed. - `zcash_client_sqlite::NoteId` has been replaced with `zcash_client_sqlite::ReceivedNoteId` - as the `SentNoteId` variant of is now unused following changes to + as the `SentNoteId` variant of is now unused following changes to `zcash_client_backend::data_api::WalletRead`. - `zcash_client_sqlite::wallet::init::{init_blocks_table, init_accounts_table}` have been removed. `zcash_client_backend::data_api::WalletWrite::create_account` diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index d7682aed20..54d00639bb 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -66,6 +66,9 @@ pub enum SqliteClientError { /// The space of allocatable diversifier indices has been exhausted for the given account. DiversifierIndexOutOfRange, + /// The account for which information was requested does not belong to the wallet. + AccountUnknown(AccountId), + /// An error occurred deriving a spending key from a seed and an account /// identifier. KeyDerivationError(AccountId), @@ -132,6 +135,8 @@ impl fmt::Display for SqliteClientError { SqliteClientError::BlockConflict(h) => write!(f, "A block hash conflict occurred at height {}; rewind required.", u32::from(*h)), SqliteClientError::NonSequentialBlocks => write!(f, "`put_blocks` requires that the provided block range be sequential"), SqliteClientError::DiversifierIndexOutOfRange => write!(f, "The space of available diversifier indices is exhausted"), + SqliteClientError::AccountUnknown(id) => write!(f, "Account {} does not belong to this wallet.", u32::from(*id)), + SqliteClientError::KeyDerivationError(acct_id) => write!(f, "Key derivation failed for account {:?}", acct_id), SqliteClientError::AccountIdDiscontinuity => write!(f, "Wallet account identifiers must be sequential."), SqliteClientError::AccountIdOutOfRange => write!(f, "Wallet account identifiers must be less than 0x7FFFFFFF."), diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 829abbbe63..318fc16abc 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -206,6 +206,14 @@ impl, P: consensus::Parameters> WalletRead for W wallet::get_tx_height(self.conn.borrow(), txid).map_err(SqliteClientError::from) } + fn get_wallet_birthday(&self) -> Result, Self::Error> { + wallet::wallet_birthday(self.conn.borrow()).map_err(SqliteClientError::from) + } + + fn get_account_birthday(&self, account: AccountId) -> Result { + wallet::account_birthday(self.conn.borrow(), account).map_err(SqliteClientError::from) + } + fn get_current_address( &self, account: AccountId, diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index a2bf0433e9..4cd5423c6d 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -675,6 +675,40 @@ pub(crate) fn get_sent_memo( .transpose() } +/// Returns the minimum birthday height for accounts in the wallet. +/// +/// TODO ORCHARD: we should consider whether we want to permit protocol-restricted accounts; if so, +/// we would then want this method to take a protocol identifier to be able to learn the wallet's +/// "Orchard birthday" which might be different from the overall wallet birthday. +pub(crate) fn wallet_birthday( + conn: &rusqlite::Connection, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT MIN(birthday_height) AS wallet_birthday FROM accounts", + [], + |row| { + row.get::<_, Option>(0) + .map(|opt| opt.map(BlockHeight::from)) + }, + ) +} + +pub(crate) fn account_birthday( + conn: &rusqlite::Connection, + account: AccountId, +) -> Result { + conn.query_row( + "SELECT birthday_height + FROM accounts + WHERE account = :account_id", + named_params![":account_id": u32::from(account)], + |row| row.get::<_, u32>(0).map(BlockHeight::from), + ) + .optional() + .map_err(SqliteClientError::from) + .and_then(|opt| opt.ok_or(SqliteClientError::AccountUnknown(account))) +} + /// Returns the minimum and maximum heights for blocks stored in the wallet database. pub(crate) fn block_height_extrema( conn: &rusqlite::Connection, From a51712830fc305630ade5c47a96540c404c7cbc4 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 23 Aug 2023 15:04:24 -0600 Subject: [PATCH 0130/1122] zcash_client_sqlite: ignore unscanned ranges below the birthday height in note selection --- zcash_client_sqlite/src/wallet/sapling.rs | 36 +++++++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 564a785c49..099e6d6847 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -20,7 +20,7 @@ use zcash_client_backend::{ use crate::{error::SqliteClientError, ReceivedNoteId}; -use super::memo_repr; +use super::{memo_repr, wallet_birthday}; /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedSaplingOutput { @@ -132,6 +132,15 @@ pub(crate) fn get_spendable_sapling_notes( anchor_height: BlockHeight, exclude: &[ReceivedNoteId], ) -> Result>, SqliteClientError> { + let birthday_height = match wallet_birthday(conn)? { + Some(birthday) => birthday, + None => { + // the wallet birthday can only be unknown if there are no accounts in the wallet; in + // such a case, the wallet has no notes to spend. + return Ok(vec![]); + } + }; + let mut stmt_unscanned_tip = conn.prepare_cached( "SELECT 1 FROM v_sapling_shard_unscanned_ranges WHERE :anchor_height BETWEEN subtree_start_height AND IFNULL(subtree_end_height, :anchor_height) @@ -159,8 +168,10 @@ pub(crate) fn get_spendable_sapling_notes( -- select all the unscanned ranges involving the shard containing this note WHERE sapling_received_notes.commitment_tree_position >= unscanned.start_position AND sapling_received_notes.commitment_tree_position < unscanned.end_position_exclusive - -- exclude unscanned ranges above the anchor height which don't affect spendability + -- exclude unscanned ranges that start above the anchor height (they don't affect spendability) AND unscanned.block_range_start <= :anchor_height + -- exclude unscanned ranges that end below the wallet birthday + AND unscanned.block_range_end > :wallet_birthday )", )?; @@ -169,9 +180,10 @@ pub(crate) fn get_spendable_sapling_notes( let notes = stmt_select_notes.query_and_then( named_params![ - ":account": &u32::from(account), - ":anchor_height": &u32::from(anchor_height), + ":account": u32::from(account), + ":anchor_height": u32::from(anchor_height), ":exclude": &excluded_ptr, + ":wallet_birthday": u32::from(birthday_height) ], to_spendable_note, )?; @@ -186,6 +198,15 @@ pub(crate) fn select_spendable_sapling_notes( anchor_height: BlockHeight, exclude: &[ReceivedNoteId], ) -> Result>, SqliteClientError> { + let birthday_height = match wallet_birthday(conn)? { + Some(birthday) => birthday, + None => { + // the wallet birthday can only be unknown if there are no accounts in the wallet; in + // such a case, the wallet has no notes to spend. + return Ok(vec![]); + } + }; + let mut stmt_unscanned_tip = conn.prepare_cached( "SELECT 1 FROM v_sapling_shard_unscanned_ranges WHERE :anchor_height BETWEEN subtree_start_height AND IFNULL(subtree_end_height, :anchor_height) @@ -233,8 +254,10 @@ pub(crate) fn select_spendable_sapling_notes( -- select all the unscanned ranges involving the shard containing this note WHERE sapling_received_notes.commitment_tree_position >= unscanned.start_position AND sapling_received_notes.commitment_tree_position < unscanned.end_position_exclusive - -- exclude unscanned ranges above the anchor height which don't affect spendability + -- exclude unscanned ranges that start above the anchor height (they don't affect spendability) AND unscanned.block_range_start <= :anchor_height + -- exclude unscanned ranges that end below the wallet birthday + AND unscanned.block_range_end > :wallet_birthday ) ) SELECT id_note, diversifier, value, rcm, commitment_tree_position @@ -252,7 +275,8 @@ pub(crate) fn select_spendable_sapling_notes( ":account": &u32::from(account), ":anchor_height": &u32::from(anchor_height), ":target_value": &i64::from(target_value), - ":exclude": &excluded_ptr + ":exclude": &excluded_ptr, + ":wallet_birthday": u32::from(birthday_height) ], to_spendable_note, )?; From 5b3f54410261d610d9090edb8dcddd5b4b26ddda Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 1 Sep 2023 10:06:19 -0600 Subject: [PATCH 0131/1122] Apply suggestions from code review Co-authored-by: str4d Co-authored-by: Daira Emma Hopwood --- zcash_client_backend/src/data_api.rs | 24 ++++++---- zcash_client_backend/src/proto.rs | 1 + zcash_client_sqlite/CHANGELOG.md | 2 +- zcash_client_sqlite/src/wallet.rs | 20 +++++---- .../init/migrations/add_account_birthdays.rs | 45 +++++++++---------- zcash_client_sqlite/src/wallet/scanning.rs | 25 ++++------- 6 files changed, 60 insertions(+), 57 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 3ef4eb2afc..2211318e47 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -121,8 +121,8 @@ pub trait WalletRead { /// Returns the birthday height for the wallet. /// - /// This returns earliest birthday height among accounts maintained by this wallet, or - /// `Ok(None)` if the wallet has no initialized accounts. + /// This returns the earliest birthday height among accounts maintained by this wallet, + /// or `Ok(None)` if the wallet has no initialized accounts. fn get_wallet_birthday(&self) -> Result, Self::Error>; /// Returns the birthday height for the given account, or an error if the account is not known @@ -487,7 +487,7 @@ pub struct AccountBirthday { recover_until: Option, } -/// Errors that can occur in the construction of an [`AccountBirthday`] from a [`TreeState`] +/// Errors that can occur in the construction of an [`AccountBirthday`] from a [`TreeState`]. pub enum BirthdayError { HeightInvalid(TryFromIntError), Decode(io::Error), @@ -509,7 +509,7 @@ impl AccountBirthday { /// Constructs a new [`AccountBirthday`] from its constituent parts. /// /// * `height`: The birthday height of the account. This is defined as the height of the first - /// block block to be scanned in wallet recovery. + /// block to be scanned in wallet recovery. /// * `sapling_frontier`: The Sapling note commitment tree frontier as of the end of the block /// prior to the birthday height. /// * `recover_until`: An optional height at which the wallet should exit "recovery mode". In @@ -572,6 +572,12 @@ impl AccountBirthday { } #[cfg(feature = "test-dependencies")] + /// Constructs a new [`AccountBirthday`] at Sapling activation, with no + /// "recover until" height. + /// + /// # Panics + /// + /// Panics if the Sapling activation height is not set. pub fn from_sapling_activation( params: &P, ) -> AccountBirthday { @@ -597,12 +603,12 @@ pub trait WalletWrite: WalletRead { /// Returns the account identifier for the newly-created wallet database entry, along with the /// associated [`UnifiedSpendingKey`]. /// - /// If a birthday height is having a height is below the current chain tip, this operation will + /// If `birthday.height()` is below the current chain tip, this operation will /// trigger a re-scan of the blocks at and above the provided height. The birthday height is /// defined as the minimum block height that will be scanned for funds belonging to the wallet. /// /// For new wallets, callers should construct the [`AccountBirthday`] using - /// [`AccountBirthday::from_treestate`] for the block at height `chain_tip_height - PRUNING_DEPTH`. + /// [`AccountBirthday::from_treestate`] for the block at height `chain_tip_height - 100`. /// Setting the birthday height to a tree state below the pruning depth ensures that reorgs /// cannot cause funds intended for the wallet to be missed; otherwise, if the chain tip height /// were used for the wallet birthday, a transaction targeted at a height greater than the @@ -612,9 +618,9 @@ pub trait WalletWrite: WalletRead { /// wallet state, you should use this method to add all of the desired accounts before scanning /// the chain from the seed's birthday height. /// - /// By convention, wallets should only allow a new account to be generated after funds have - /// been received by the currently-available account (in order to enable automated account - /// recovery). + /// By convention, wallets should only allow a new account to be generated after confirmed + /// funds have been received by the currently-available account (in order to enable automated + /// account recovery). /// /// [ZIP 316]: https://zips.z.cash/zip-0316 fn create_account( diff --git a/zcash_client_backend/src/proto.rs b/zcash_client_backend/src/proto.rs index 711faface6..ee21a8e9fa 100644 --- a/zcash_client_backend/src/proto.rs +++ b/zcash_client_backend/src/proto.rs @@ -147,6 +147,7 @@ impl compact_formats::CompactSaplingSpend { } impl service::TreeState { + /// Deserializes and returns the Sapling note commitment tree field of the tree state. pub fn sapling_tree(&self) -> io::Result> { let sapling_tree_bytes = hex::decode(&self.sapling_tree).map_err(|e| { io::Error::new( diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 33b7e128e0..378e0381f9 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -38,7 +38,7 @@ and this library adheres to Rust's notion of ### Removed - The empty `wallet::transact` module has been removed. - `zcash_client_sqlite::NoteId` has been replaced with `zcash_client_sqlite::ReceivedNoteId` - as the `SentNoteId` variant of is now unused following changes to + as the `SentNoteId` variant is now unused following changes to `zcash_client_backend::data_api::WalletRead`. - `zcash_client_sqlite::wallet::init::{init_blocks_table, init_accounts_table}` have been removed. `zcash_client_backend::data_api::WalletWrite::create_account` diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 4cd5423c6d..e8b1c0e293 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -196,7 +196,11 @@ pub(crate) fn add_account( shard_tree.insert_frontier_nodes( frontier.clone(), Retention::Checkpoint { - // This subtraction is safe, because the non-empty frontier cannot exist at genesis + // This subtraction is safe, because all leaves in the tree appear in blocks, and + // the invariant that birthday.height() always corresponds to the block for which + // `frontier` is the tree state at the start of the block. Together, this means + // there exists a prior block for which frontier is the tree state at the end of + // the block. id: birthday.height() - 1, is_marked: false, }, @@ -207,7 +211,7 @@ pub(crate) fn add_account( .activation_height(NetworkUpgrade::Sapling) .expect("Sapling activation height must be available."); - // Add the ignored range up to and including the birthday height. + // Add the ignored range up to the birthday height. if sapling_activation_height < birthday.height() { let ignored_range = sapling_activation_height..birthday.height(); @@ -223,8 +227,8 @@ pub(crate) fn add_account( )?; }; - // Rewrite the scan ranges above the birthday height so that we'll ensure we re-scan to find - // any notes that might belong to the newly added account. + // Rewrite the scan ranges starting from the birthday height so that we'll ensure we + // re-scan to find any notes that might belong to the newly added account. if let Some(t) = chain_tip { let rescan_range = birthday.height()..(t + 1); @@ -676,10 +680,10 @@ pub(crate) fn get_sent_memo( } /// Returns the minimum birthday height for accounts in the wallet. -/// -/// TODO ORCHARD: we should consider whether we want to permit protocol-restricted accounts; if so, -/// we would then want this method to take a protocol identifier to be able to learn the wallet's -/// "Orchard birthday" which might be different from the overall wallet birthday. +// +// TODO ORCHARD: we should consider whether we want to permit protocol-restricted accounts; if so, +// we would then want this method to take a protocol identifier to be able to learn the wallet's +// "Orchard birthday" which might be different from the overall wallet birthday. pub(crate) fn wallet_birthday( conn: &rusqlite::Connection, ) -> Result, rusqlite::Error> { diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_account_birthdays.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_account_birthdays.rs index bf1e5e380d..e1893362ed 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_account_birthdays.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_account_birthdays.rs @@ -36,29 +36,28 @@ impl RusqliteMigration for Migration

{ fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { transaction.execute_batch(&format!( "ALTER TABLE accounts ADD COLUMN birthday_height INTEGER; - ALTER TABLE accounts ADD COLUMN recover_until_height INTEGER; - - -- set the birthday height to the height of the first block in the blocks table - UPDATE accounts SET birthday_height = MIN(blocks.height) FROM blocks; - -- if the blocks table is empty, set the birthday height to Sapling activation - 1 - UPDATE accounts SET birthday_height = {} WHERE birthday_height IS NULL; - - CREATE TABLE accounts_new ( - account INTEGER PRIMARY KEY, - ufvk TEXT NOT NULL, - birthday_height INTEGER NOT NULL, - recover_until_height INTEGER - ); - - INSERT INTO accounts_new (account, ufvk, birthday_height) - SELECT account, ufvk, birthday_height FROM accounts; - - PRAGMA foreign_keys=OFF; - PRAGMA legacy_alter_table = ON; - DROP TABLE accounts; - ALTER TABLE accounts_new RENAME TO accounts; - PRAGMA legacy_alter_table = OFF; - PRAGMA foreign_keys=ON;", + + -- set the birthday height to the height of the first block in the blocks table + UPDATE accounts SET birthday_height = MIN(blocks.height) FROM blocks; + -- if the blocks table is empty, set the birthday height to Sapling activation - 1 + UPDATE accounts SET birthday_height = {} WHERE birthday_height IS NULL; + + CREATE TABLE accounts_new ( + account INTEGER PRIMARY KEY, + ufvk TEXT NOT NULL, + birthday_height INTEGER NOT NULL, + recover_until_height INTEGER + ); + + INSERT INTO accounts_new (account, ufvk, birthday_height) + SELECT account, ufvk, birthday_height FROM accounts; + + PRAGMA foreign_keys=OFF; + PRAGMA legacy_alter_table = ON; + DROP TABLE accounts; + ALTER TABLE accounts_new RENAME TO accounts; + PRAGMA legacy_alter_table = OFF; + PRAGMA foreign_keys=ON;", u32::from( self.params .activation_height(NetworkUpgrade::Sapling) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 69030a15a5..7d1d9ed3ca 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -1347,28 +1347,21 @@ mod tests { let sap_active = st.sapling_activation_height(); let expected = vec![ - // The range up to and including the wallet's birthday height is ignored. + // The range up to the wallet's birthday height is ignored. scan_range(u32::from(sap_active)..u32::from(birthday.height()), Ignored), ]; let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); assert_eq!(actual, expected); - // Set up some shard history + // Set up some shard root history before the wallet birthday st.wallet_mut() .put_sapling_subtree_roots( 0, - &[ - // Add the end of a commitment tree below the wallet birthday. We currently - // need to scan from this height up to the tip to make notes spendable, though - // this should not be necessary as we have added a frontier that should - // complete the left-hand side of the required shard; this can be fixed once we - // have proper account birthdays. - CommitmentTreeRoot::from_parts( - birthday.height() - 1000, - // fake a hash, the value doesn't matter - Node::empty_leaf(), - ), - ], + &[CommitmentTreeRoot::from_parts( + birthday.height() - 1000, + // fake a hash, the value doesn't matter + Node::empty_leaf(), + )], ) .unwrap(); @@ -1394,8 +1387,8 @@ mod tests { // Verify that the suggested scan ranges match what is expected let expected = vec![ - // The birthday height was "last scanned" (as the wallet birthday) so we verify 10 - // blocks starting at that height. + // The birthday height is the "first to be scanned" (as the wallet birthday), + // so we verify 10 blocks starting at that height. scan_range( u32::from(birthday.height())..u32::from(birthday.height() + 10), Verify, From 14d61a2e98627645cd360d286989f064fcd3d8ea Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 1 Sep 2023 12:46:13 -0600 Subject: [PATCH 0132/1122] zcash_primitives: Remove the deprecated 1000-ZAT DEFAULT_FEE constant --- zcash_primitives/CHANGELOG.md | 2 ++ zcash_primitives/src/transaction/components/amount.rs | 8 -------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 44f903ca0d..d5855b66a9 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -37,6 +37,8 @@ and this library adheres to Rust's notion of - `impl {PartialEq, Eq} for transaction::builder::Error` (use `assert_matches!` where error comparisons are required) - `zcash_primitives::transaction::components::orchard::Unauthorized` +- `zcash_primitives::transaction::components::amount::DEFAULT_FEE` was + deprecated in 0.12.0 and has now been removed. ## [0.12.0] - 2023-06-06 ### Added diff --git a/zcash_primitives/src/transaction/components/amount.rs b/zcash_primitives/src/transaction/components/amount.rs index 83b57c38d9..b4e8c4f1e5 100644 --- a/zcash_primitives/src/transaction/components/amount.rs +++ b/zcash_primitives/src/transaction/components/amount.rs @@ -8,14 +8,6 @@ use orchard::value as orchard; pub const COIN: i64 = 1_0000_0000; pub const MAX_MONEY: i64 = 21_000_000 * COIN; -#[deprecated( - since = "0.12.0", - note = "To calculate the ZIP 317 fee, use `transaction::fees::zip317::FeeRule`. -For a constant representing the minimum ZIP 317 fee, use `transaction::fees::zip317::MINIMUM_FEE`. -For the constant amount 1000 zatoshis, use `Amount::const_from_i64(1000)`." -)] -pub const DEFAULT_FEE: Amount = Amount(1000); - /// A type-safe representation of some quantity of Zcash. /// /// An Amount can only be constructed from an integer that is within the valid monetary From 9f9a1fb92f009102c342b895b0cb6d27d2825415 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sat, 2 Sep 2023 03:00:33 +0000 Subject: [PATCH 0133/1122] zcash_client_sqlite: Rework `update_chain_tip` tests to pin behaviour --- zcash_client_sqlite/src/wallet/scanning.rs | 360 +++++++++++++++++++-- 1 file changed, 330 insertions(+), 30 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 7d1d9ed3ca..3ea883cd0c 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -834,6 +834,7 @@ mod tests { use incrementalmerkletree::{frontier::Frontier, Hashable, Level, Position}; + use secrecy::SecretVec; use zcash_client_backend::data_api::{ chain::CommitmentTreeRoot, scanning::{ScanPriority, ScanRange}, @@ -844,12 +845,14 @@ mod tests { consensus::{BlockHeight, NetworkUpgrade, Parameters}, sapling::Node, transaction::components::Amount, + zip32::DiversifiableFullViewingKey, }; use crate::{ error::SqliteClientError, - testing::{AddressType, TestBuilder}, + testing::{AddressType, BlockCache, TestBuilder, TestState}, wallet::scanning::{insert_queue_entries, replace_queue_entries, suggest_scan_ranges}, + VERIFY_LOOKAHEAD, }; use super::{join_nonoverlapping, Joined, RangeOrdering, SpanningTree}; @@ -1047,6 +1050,30 @@ mod tests { ); } + #[test] + fn spanning_tree_insert_gaps() { + use ScanPriority::*; + + let t = spanning_tree(&[(0..3, Historic), (6..8, ChainTip)]).unwrap(); + + assert_eq!( + t.into_vec(), + vec![scan_range(0..6, Historic), scan_range(6..8, ChainTip),] + ); + + let t = spanning_tree(&[(0..3, Historic), (3..4, Verify), (6..8, ChainTip)]).unwrap(); + + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..3, Historic), + scan_range(3..4, Verify), + scan_range(4..6, Historic), + scan_range(6..8, ChainTip), + ] + ); + } + #[test] fn spanning_tree_insert_rfd_span() { use ScanPriority::*; @@ -1321,11 +1348,13 @@ mod tests { ); } - #[test] - fn create_account_creates_ignored_range() { - use ScanPriority::*; - - let mut st = TestBuilder::new() + fn test_with_canopy_birthday() -> ( + TestState, + DiversifiableFullViewingKey, + AccountBirthday, + u32, + ) { + let st = TestBuilder::new() .with_block_cache() .with_test_account(|network| { // We use Canopy activation as an arbitrary birthday height that's greater than Sapling @@ -1346,27 +1375,190 @@ mod tests { let dfvk = st.test_account_sapling().unwrap(); let sap_active = st.sapling_activation_height(); + (st, dfvk, birthday, sap_active.into()) + } + + #[test] + fn create_account_creates_ignored_range() { + use ScanPriority::*; + + let (st, _, birthday, sap_active) = test_with_canopy_birthday(); + let birthday_height = birthday.height().into(); + + let expected = vec![ + // The range up to the wallet's birthday height is ignored. + scan_range(sap_active..birthday_height, Ignored), + ]; + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, expected); + } + + #[test] + fn update_chain_tip_before_create_account() { + use ScanPriority::*; + + let mut st = TestBuilder::new().with_block_cache().build(); + let sap_active = st.sapling_activation_height(); + + // Update the chain tip. + let new_tip = sap_active + 1000; + st.wallet_mut().update_chain_tip(new_tip).unwrap(); + let chain_end = u32::from(new_tip + 1); + let expected = vec![ + // The range up to the chain end is ignored. + scan_range(sap_active.into()..chain_end, Ignored), + ]; + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, expected); + + // Now add an account. + let wallet_birthday = sap_active + 500; + st.wallet_mut() + .create_account( + &SecretVec::new(vec![0; 32]), + AccountBirthday::from_parts(wallet_birthday, Frontier::empty(), None), + ) + .unwrap(); + + let expected = vec![ + // The account's birthday onward is marked for recovery. + scan_range(wallet_birthday.into()..chain_end, Historic), // The range up to the wallet's birthday height is ignored. - scan_range(u32::from(sap_active)..u32::from(birthday.height()), Ignored), + scan_range(sap_active.into()..wallet_birthday.into(), Ignored), + ]; + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, expected); + } + + #[test] + fn update_chain_tip_with_no_subtree_roots() { + use ScanPriority::*; + + let (mut st, _, birthday, sap_active) = test_with_canopy_birthday(); + + // Set up the following situation: + // + // prior_tip new_tip + // |<--- 500 --->| + // wallet_birthday + let prior_tip = birthday.height(); + let wallet_birthday = birthday.height().into(); + + // Update the chain tip. + let new_tip = prior_tip + 500; + st.wallet_mut().update_chain_tip(new_tip).unwrap(); + let chain_end = u32::from(new_tip + 1); + + // Verify that the suggested scan ranges match what is expected. + let expected = vec![ + // The wallet's birthday onward is marked for recovery. + scan_range(wallet_birthday..chain_end, Historic), + // The range below the wallet's birthday height is ignored. + scan_range(sap_active..wallet_birthday, Ignored), + ]; + + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, expected); + } + + #[test] + fn update_chain_tip_when_never_scanned() { + use ScanPriority::*; + + let (mut st, _, birthday, sap_active) = test_with_canopy_birthday(); + + // Set up the following situation: + // + // last_shard_start prior_tip new_tip + // |<----- 1000 ----->|<--- 500 --->| + // wallet_birthday + let prior_tip_height = birthday.height(); + + // Set up some shard root history before the wallet birthday. + let last_shard_start = birthday.height() - 1000; + st.wallet_mut() + .put_sapling_subtree_roots( + 0, + &[CommitmentTreeRoot::from_parts( + last_shard_start, + // fake a hash, the value doesn't matter + Node::empty_leaf(), + )], + ) + .unwrap(); + + // Update the chain tip. + let tip_height = prior_tip_height + 500; + st.wallet_mut().update_chain_tip(tip_height).unwrap(); + let chain_end = u32::from(tip_height + 1); + + // Verify that the suggested scan ranges match what is expected. + let expected = vec![ + // The entire last (incomplete) shard's range is marked for catching up to the + // chain tip, to ensure that if any notes are discovered after the wallet's + // birthday, they will be spendable. + scan_range(last_shard_start.into()..chain_end, ChainTip), + // The range below the last shard is ignored. + scan_range(sap_active..last_shard_start.into(), Ignored), ]; + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); assert_eq!(actual, expected); + } - // Set up some shard root history before the wallet birthday + #[test] + fn update_chain_tip_unstable_max_scanned() { + use ScanPriority::*; + + let (mut st, dfvk, birthday, sap_active) = test_with_canopy_birthday(); + + // Set up the following situation: + // + // prior_tip new_tip + // |<--- 500 --->|<- 40 ->|<-- 70 -->|<- 20 ->| + // wallet_birthday max_scanned last_shard_start + // + let max_scanned = birthday.height() + 500; + let prior_tip = max_scanned + 40; + + // Set up some shard root history before the wallet birthday. + let second_to_last_shard_start = birthday.height() - 1000; st.wallet_mut() .put_sapling_subtree_roots( 0, &[CommitmentTreeRoot::from_parts( - birthday.height() - 1000, + second_to_last_shard_start, // fake a hash, the value doesn't matter Node::empty_leaf(), )], ) .unwrap(); + // Set up prior chain state. This simulates us having imported a wallet + // with a birthday 520 blocks below the chain tip. + st.wallet_mut().update_chain_tip(prior_tip).unwrap(); + + // Verify that the suggested scan ranges match what is expected. + let expected = vec![ + // The second-to-last shard is currently the last shard, so it is marked for + // scanning to catch up to the prior chain tip. This includes heights prior to + // the wallet's birthday, because the wallet doesn't know that it already has + // enough data from the initial frontier to avoid having to scan this range. + scan_range( + second_to_last_shard_start.into()..(prior_tip + 1).into(), + ChainTip, + ), + // The range below the second-to-last shard is ignored. + scan_range(sap_active..second_to_last_shard_start.into(), Ignored), + ]; + + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, expected); + + // Now, scan the max scanned block. st.generate_block_at( - birthday.height(), + max_scanned, BlockHash([0u8; 32]), &dfvk, AddressType::DefaultExternal, @@ -1375,42 +1567,150 @@ mod tests { .try_into() .unwrap(), ); - st.scan_cached_blocks(birthday.height(), 1); + st.scan_cached_blocks(max_scanned, 1); - // Update the chain tip - let tip_height = st - .wallet() - .params - .activation_height(NetworkUpgrade::Nu5) + // Now simulate shutting down, and then restarting 90 blocks later, after a shard + // has been completed. + let last_shard_start = prior_tip + 70; + st.wallet_mut() + .put_sapling_subtree_roots( + 0, + &[CommitmentTreeRoot::from_parts( + last_shard_start, + // fake a hash, the value doesn't matter + Node::empty_leaf(), + )], + ) .unwrap(); - st.wallet_mut().update_chain_tip(tip_height).unwrap(); + + let new_tip = last_shard_start + 20; + st.wallet_mut().update_chain_tip(new_tip).unwrap(); + let chain_end = u32::from(new_tip + 1); // Verify that the suggested scan ranges match what is expected let expected = vec![ - // The birthday height is the "first to be scanned" (as the wallet birthday), - // so we verify 10 blocks starting at that height. + // The max scanned block's connectivity is verified by scanning the next 10 blocks. scan_range( - u32::from(birthday.height())..u32::from(birthday.height() + 10), + (max_scanned + 1).into()..(max_scanned + 1 + VERIFY_LOOKAHEAD).into(), Verify, ), - // The remainder of the shard after the verify segment is required in order to make - // notes spendable, so it has priority `ChainTip` + // The last shard needs to catch up to the chain tip in order to make notes spendable. + scan_range(last_shard_start.into()..chain_end, ChainTip), + // The range between the verification blocks and the prior tip is still in the queue. scan_range( - u32::from(birthday.height() + 10)..u32::from(tip_height + 1), + (max_scanned + 1 + VERIFY_LOOKAHEAD).into()..(prior_tip + 1).into(), ChainTip, ), - // The remainder of the shard prior to the birthday height must be scanned because the - // wallet doesn't know that it already has enough data from the initial frontier to - // avoid having to scan this range. + // The remainder of the second-to-last shard's range is still in the queue. + scan_range( + second_to_last_shard_start.into()..max_scanned.into(), + ChainTip, + ), + // The gap between the prior tip and the last shard is deferred as low priority. + scan_range((prior_tip + 1).into()..last_shard_start.into(), Historic), + // The max scanned block itself is left as-is. + scan_range(max_scanned.into()..(max_scanned + 1).into(), Scanned), + // The range below the second-to-last shard is ignored. + scan_range(sap_active..second_to_last_shard_start.into(), Ignored), + ]; + + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, expected); + } + + #[test] + fn update_chain_tip_stable_max_scanned() { + use ScanPriority::*; + + let (mut st, dfvk, birthday, sap_active) = test_with_canopy_birthday(); + + // Set up the following situation: + // + // prior_tip new_tip + // |<--- 500 --->|<- 20 ->|<-- 50 -->|<- 20 ->| + // wallet_birthday max_scanned last_shard_start + // + let max_scanned = birthday.height() + 500; + let prior_tip = max_scanned + 20; + + // Set up some shard root history before the wallet birthday. + let second_to_last_shard_start = birthday.height() - 1000; + st.wallet_mut() + .put_sapling_subtree_roots( + 0, + &[CommitmentTreeRoot::from_parts( + second_to_last_shard_start, + // fake a hash, the value doesn't matter + Node::empty_leaf(), + )], + ) + .unwrap(); + + // Set up prior chain state. This simulates us having imported a wallet + // with a birthday 520 blocks below the chain tip. + st.wallet_mut().update_chain_tip(prior_tip).unwrap(); + + // Verify that the suggested scan ranges match what is expected. + let expected = vec![ + // The second-to-last shard is currently the last shard, so it is marked for + // scanning to catch up to the prior chain tip. This includes heights prior to + // the wallet's birthday, because the wallet doesn't know that it already has + // enough data from the initial frontier to avoid having to scan this range. scan_range( - u32::from(birthday.height() - 1000)..u32::from(birthday.height()), + second_to_last_shard_start.into()..(prior_tip + 1).into(), ChainTip, ), - // The range below the wallet's birthday height is ignored + // The range below the second-to-last shard is ignored. + scan_range(sap_active..second_to_last_shard_start.into(), Ignored), + ]; + + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, expected); + + // Now, scan the max scanned block. + st.generate_block_at( + max_scanned, + BlockHash([0u8; 32]), + &dfvk, + AddressType::DefaultExternal, + Amount::const_from_i64(10000), + u64::from(birthday.sapling_frontier().value().unwrap().position() + 1) + .try_into() + .unwrap(), + ); + st.scan_cached_blocks(max_scanned, 1); + + // Now simulate shutting down, and then restarting 70 blocks later, after a shard + // has been completed. + let last_shard_start = prior_tip + 50; + st.wallet_mut() + .put_sapling_subtree_roots( + 0, + &[CommitmentTreeRoot::from_parts( + last_shard_start, + // fake a hash, the value doesn't matter + Node::empty_leaf(), + )], + ) + .unwrap(); + + let new_tip = last_shard_start + 20; + st.wallet_mut().update_chain_tip(new_tip).unwrap(); + let chain_end = u32::from(new_tip + 1); + + // Verify that the suggested scan ranges match what is expected. + let expected = vec![ + // The blocks after the max scanned block up to the chain tip are prioritised. + scan_range((max_scanned + 1).into()..chain_end, ChainTip), + // The remainder of the second-to-last shard's range is still in the queue. scan_range( - u32::from(sap_active)..u32::from(birthday.height() - 1000), - Ignored, + second_to_last_shard_start.into()..max_scanned.into(), + ChainTip, ), + // The max scanned block itself is left as-is. + scan_range(max_scanned.into()..(max_scanned + 1).into(), Scanned), + // The range below the second-to-last shard is ignored. + scan_range(sap_active..second_to_last_shard_start.into(), Ignored), ]; let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); From 81d7d9349cbfdbc1ca3ff54e5228f0e788dcbc00 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sat, 2 Sep 2023 03:01:23 +0000 Subject: [PATCH 0134/1122] zcash_client_sqlite: Fix bugs in `update_chain_tip` Co-authored-by: Kris Nuttycombe --- zcash_client_sqlite/src/wallet/scanning.rs | 193 +++++++++++++-------- 1 file changed, 118 insertions(+), 75 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 3ea883cd0c..7d8d1c79a0 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -18,6 +18,8 @@ use crate::{ PRUNING_DEPTH, VERIFY_LOOKAHEAD, }; +use super::wallet_birthday; + #[derive(Debug, Clone, Copy)] enum InsertOn { Left, @@ -715,8 +717,17 @@ pub(crate) fn update_chain_tip( params: &P, new_tip: BlockHeight, ) -> Result<(), SqliteClientError> { + // If the caller provided a chain tip that is before Sapling activation, do nothing. + let sapling_activation = match params.activation_height(NetworkUpgrade::Sapling) { + Some(h) => h, + None => return Ok(()), + }; + // Read the previous max scanned height from the blocks table - let prior_tip = block_height_extrema(conn)?.map(|(_, prior_tip)| prior_tip); + let max_scanned = block_height_extrema(conn)?.map(|(_, max_scanned)| max_scanned); + + // Read the wallet birthday (if known). + let wallet_birthday = wallet_birthday(conn)?; // If the chain tip is below the prior max scanned height, then the caller has caught // the chain in the middle of a reorg. Do nothing; the caller will continue using the @@ -730,11 +741,10 @@ pub(crate) fn update_chain_tip( // We don't check the shard height, as normal usage would have the caller update the // shard state prior to this call, so it is possible and expected to be in a situation // where we should update the tip-related scan ranges but not the shard-related ones. - if let Some(h) = prior_tip { - if new_tip < h { - return Ok(()); - } - } + match max_scanned { + Some(h) if new_tip < h => return Ok(()), + _ => (), + }; // `ScanRange` uses an exclusive upper bound. let chain_end = new_tip + 1; @@ -748,82 +758,115 @@ pub(crate) fn update_chain_tip( )?; // Create a scanning range for the fragment of the last shard leading up to new tip. - let shard_entry = shard_start_height + let tip_shard_entry = shard_start_height .filter(|h| h < &chain_end) .map(|h| ScanRange::from_parts(h..chain_end, ScanPriority::ChainTip)); - // Create scanning ranges to either validate potentially invalid blocks at the wallet's view - // of the chain tip, or connect the prior tip to the new tip. - let tip_entry = prior_tip.map(|prior_tip| { - // If we don't have shard metadata, this means we're doing linear scanning, so create a - // scan range from the prior tip to the current tip with `Historic` priority. - if shard_entry.is_none() { - ScanRange::from_parts(prior_tip..chain_end, ScanPriority::Historic) - } else { - // Determine the height to which we expect blocks retrieved from the block source to be stable - // and not subject to being reorg'ed. - let stable_height = new_tip.saturating_sub(PRUNING_DEPTH); - - // If the wallet's prior tip is above the stable height, prioritize the range between - // it and the new tip as `ChainTip`. Otherwise, prioritize the `VERIFY_LOOKAHEAD` - // blocks above the wallet's prior tip as `Verify`. Since `scan_cached_blocks` - // retrieves the metadata for the block being connected to, the connectivity to the - // prior tip will always be checked. Since `Verify` ranges have maximum priority, even - // if the block source begins downloading blocks from the shard scan range (which ends - // at the stable height) the scanner should not attempt to scan those blocks until the - // tip range has been completely checked and any required rewinds have been performed. - if prior_tip >= stable_height { - // This may overlap the `shard_entry` range and if so will be coalesced with it. - ScanRange::from_parts(prior_tip..chain_end, ScanPriority::ChainTip) + // Create scan ranges to either validate potentially invalid blocks at the wallet's + // view of the chain tip, or connect the prior tip to the new tip. + let tip_entry = max_scanned.map_or_else( + || { + // No blocks have been scanned, so we need to anchor the start of the new scan + // range to something else. + wallet_birthday.map_or_else( + // We don't have a wallet birthday, which means we have no accounts yet. + // We can therefore ignore all blocks up to the chain tip. + || ScanRange::from_parts(sapling_activation..chain_end, ScanPriority::Ignored), + // We have a wallet birthday, so mark all blocks between that and the + // chain tip as `Historic` (performing wallet recovery). + |wallet_birthday| { + ScanRange::from_parts(wallet_birthday..chain_end, ScanPriority::Historic) + }, + ) + }, + |max_scanned| { + // The scan range starts at the block after the max scanned height. Since + // `scan_cached_blocks` retrieves the metadata for the block being connected to + // (if it exists), the connectivity of the scan range to the max scanned block + // will always be checked if relevant. + let min_unscanned = max_scanned + 1; + + // If we don't have shard metadata, this means we're doing linear scanning, so + // create a scan range from the prior tip to the current tip with `Historic` + // priority. + if tip_shard_entry.is_none() { + ScanRange::from_parts(min_unscanned..chain_end, ScanPriority::Historic) } else { - // The prior tip is in the range that we now expect to be stable, so we need to verify - // and advance it up to at most the stable height. The shard entry will then cover - // the range to the new tip at the lower `ChainTip` priority. - ScanRange::from_parts( - prior_tip..min(stable_height, prior_tip + VERIFY_LOOKAHEAD), - ScanPriority::Verify, - ) + // Determine the height to which we expect new blocks retrieved from the + // block source to be stable and not subject to being reorg'ed. + let stable_height = new_tip.saturating_sub(PRUNING_DEPTH); + + // If the wallet's max scanned height is above the stable height, + // prioritize the range between it and the new tip as `ChainTip`. + if max_scanned > stable_height { + // We are in the steady-state case, where a wallet is close to the + // chain tip and just needs to catch up. + // + // This overlaps the `tip_shard_entry` range and so will be coalesced + // with it. + ScanRange::from_parts(min_unscanned..chain_end, ScanPriority::ChainTip) + } else { + // In this case, the max scanned height is considered stable relative + // to the chain tip. However, it may be stable or unstable relative to + // the prior chain tip, which we could determine by looking up the + // prior chain tip height from the scan queue. For simplicity we merge + // these two cases together, and pretend that the max scanned block is + // always unstable relative to the prior chain tip. + // + // To confirm its stability, prioritize the `VERIFY_LOOKAHEAD` blocks + // above the max scanned height as `Verify`: + // + // - We use `Verify` to ensure this connectivity check is performed, + // along with any required rewinds, before any `ChainTip` ranges + // (from this or any prior `update_chain_tip` call) are scanned. + // + // - We prioritize `VERIFY_LOOKAHEAD` blocks because this is expected + // to be 12.5 minutes, within which it is reasonable for a user to + // have potentially received a transaction (if they opened their + // wallet to provide an address to someone else, or spent their own + // funds creating a change output), without necessarily having left + // their wallet open long enough for the transaction to be mined and + // the corresponding block to be scanned. + // + // - We limit the range to at most the stable region, to prevent any + // `Verify` ranges from being susceptible to reorgs, and potentially + // interfering with subsequent `Verify` ranges defined by future + // calls to `update_chain_tip`. Any gap between `stable_height` and + // `shard_start_height` will be filled by the scan range merging + // logic with a `Historic` range. + // + // If `max_scanned == stable_height` then this is a zero-length range. + // In this case, any non-empty `(stable_height+1)..shard_start_height` + // will be marked `Historic`, minimising the prioritised blocks at the + // chain tip and allowing for other ranges (for example, `FoundNote`) + // to take priority. + ScanRange::from_parts( + min_unscanned..min(stable_height + 1, min_unscanned + VERIFY_LOOKAHEAD), + ScanPriority::Verify, + ) + } } - } - }); - if let Some(entry) = &shard_entry { + }, + ); + if let Some(entry) = &tip_shard_entry { debug!("{} will update latest shard", entry); } - if let Some(entry) = &tip_entry { - debug!("{} will connect prior tip to new tip", entry); - } - - let query_range = match (shard_entry.as_ref(), tip_entry.as_ref()) { - (Some(se), Some(te)) => Some(Range { - start: min(se.block_range().start, te.block_range().start), - end: max(se.block_range().end, te.block_range().end), - }), - (Some(se), None) => Some(se.block_range().clone()), - (None, Some(te)) => Some(te.block_range().clone()), - (None, None) => None, + debug!("{} will connect prior scanned state to new tip", tip_entry); + + let query_range = match tip_shard_entry.as_ref() { + Some(se) => Range { + start: min(se.block_range().start, tip_entry.block_range().start), + end: max(se.block_range().end, tip_entry.block_range().end), + }, + None => tip_entry.block_range().clone(), }; - if let Some(query_range) = query_range { - replace_queue_entries::( - conn, - &query_range, - shard_entry.into_iter().chain(tip_entry.into_iter()), - false, - )?; - } else { - // If we have neither shard data nor any existing block data in the database, we should also - // have no existing scan queue entries and can fall back to linear scanning from Sapling - // activation. - if let Some(sapling_activation) = params.activation_height(NetworkUpgrade::Sapling) { - // If the caller provided a chain tip that is before Sapling activation, do - // nothing. - if sapling_activation < chain_end { - let scan_range = - ScanRange::from_parts(sapling_activation..chain_end, ScanPriority::Historic); - insert_queue_entries(conn, Some(scan_range).iter())?; - } - } - } + replace_queue_entries::( + conn, + &query_range, + tip_shard_entry.into_iter().chain(Some(tip_entry)), + false, + )?; Ok(()) } @@ -1341,8 +1384,8 @@ mod tests { assert_matches!( st.wallet().suggest_scan_ranges(), Ok(scan_ranges) if scan_ranges == vec![ - scan_range((sap_active + 319)..(sap_active + 329), Verify), - scan_range((sap_active + 329)..(sap_active + 451), ChainTip), + scan_range((sap_active + 320)..(sap_active + 330), Verify), + scan_range((sap_active + 330)..(sap_active + 451), ChainTip), scan_range((sap_active + 300)..(sap_active + 310), ChainTip) ] ); From 75d4f28b9ced14ffa4491483b63c4afb7fe04c3c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Sep 2023 05:54:36 +0000 Subject: [PATCH 0135/1122] build(deps): bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/book.yml | 2 +- .github/workflows/ci.yml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index f62273e552..b92f4de921 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -9,7 +9,7 @@ jobs: deploy: runs-on: ubuntu-18.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@nightly id: toolchain - run: rustup override set ${{steps.toolchain.outputs.name}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36ac21b7ae..3d03e3d29f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: os: [ubuntu-latest, windows-latest, macOS-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Fetch path to Zcash parameters working-directory: ./zcash_proofs shell: bash @@ -43,7 +43,7 @@ jobs: - wasm32-wasi steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Add target run: rustup target add ${{ matrix.target }} - run: cargo fetch @@ -58,7 +58,7 @@ jobs: name: Bitrot check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Build benchmarks to prevent bitrot - name: Build benchmarks run: cargo build --all --benches @@ -68,7 +68,7 @@ jobs: timeout-minutes: 30 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Run clippy uses: actions-rs/clippy-check@v1 with: @@ -82,7 +82,7 @@ jobs: runs-on: ubuntu-latest continue-on-error: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@beta id: toolchain - run: rustup override set ${{steps.toolchain.outputs.name}} @@ -102,7 +102,7 @@ jobs: options: --security-opt seccomp=unconfined steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Fetch path to Zcash parameters working-directory: ./zcash_proofs shell: bash @@ -127,7 +127,7 @@ jobs: name: Intra-doc links runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: cargo fetch # Requires #![deny(rustdoc::broken_intra_doc_links)] in crates. - name: Check intra-doc links @@ -138,6 +138,6 @@ jobs: timeout-minutes: 30 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check formatting run: cargo fmt --all -- --check From afec2ee218fe2dceded3fa2d18f6e67f9d8a1f4f Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 5 Sep 2023 14:51:09 +0000 Subject: [PATCH 0136/1122] `zcash_client_sqlite`: Fix refactor bug: ignore pre-Sapling chain tips The pre-refactor code did this correctly, but the comment was moved without fully moving the logic. --- zcash_client_sqlite/src/wallet/scanning.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 7d8d1c79a0..00632cfa8c 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -719,8 +719,8 @@ pub(crate) fn update_chain_tip( ) -> Result<(), SqliteClientError> { // If the caller provided a chain tip that is before Sapling activation, do nothing. let sapling_activation = match params.activation_height(NetworkUpgrade::Sapling) { - Some(h) => h, - None => return Ok(()), + Some(h) if h <= new_tip => h, + _ => return Ok(()), }; // Read the previous max scanned height from the blocks table From f132af1f3350813f185a2c5c1eaf31a659785fe5 Mon Sep 17 00:00:00 2001 From: str4d Date: Tue, 5 Sep 2023 15:54:15 +0100 Subject: [PATCH 0137/1122] Documentation fixes Co-authored-by: Daira Emma Hopwood --- zcash_client_sqlite/src/wallet/scanning.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 00632cfa8c..640f6bf9b6 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -810,13 +810,13 @@ pub(crate) fn update_chain_tip( // to the chain tip. However, it may be stable or unstable relative to // the prior chain tip, which we could determine by looking up the // prior chain tip height from the scan queue. For simplicity we merge - // these two cases together, and pretend that the max scanned block is - // always unstable relative to the prior chain tip. + // these two cases together, and proceed as though the max scanned + // block is unstable relative to the prior chain tip. // // To confirm its stability, prioritize the `VERIFY_LOOKAHEAD` blocks // above the max scanned height as `Verify`: // - // - We use `Verify` to ensure this connectivity check is performed, + // - We use `Verify` to ensure that a connectivity check is performed, // along with any required rewinds, before any `ChainTip` ranges // (from this or any prior `update_chain_tip` call) are scanned. // From dd60f51d3cc4c03430fe1d7c96462df0251b3856 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 5 Sep 2023 09:22:38 -0600 Subject: [PATCH 0138/1122] Address unresovled code review comments from #907 --- zcash_client_backend/CHANGELOG.md | 5 ++-- zcash_client_sqlite/src/error.rs | 4 +-- zcash_client_sqlite/src/wallet.rs | 13 +++------ zcash_client_sqlite/src/wallet/scanning.rs | 34 +++++++++++----------- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 80042ceade..683f238158 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -14,8 +14,9 @@ and this library adheres to Rust's notion of birthday. The account birthday is defined as the minimum height among blocks to be scanned when recovering an account. - Account creation now requires the caller to provide account birthday information, - including the state of the note commitment tree at the end of the block prior to - the birthday height. + including the state of the note commitment tree at the end of the block prior + to the birthday height. A wallet's birthday is the earliest birthday height + among accounts maintained by the wallet. ### Added - `impl Eq for zcash_client_backend::address::RecipientAddress` diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 54d00639bb..b513fc3d12 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -135,9 +135,9 @@ impl fmt::Display for SqliteClientError { SqliteClientError::BlockConflict(h) => write!(f, "A block hash conflict occurred at height {}; rewind required.", u32::from(*h)), SqliteClientError::NonSequentialBlocks => write!(f, "`put_blocks` requires that the provided block range be sequential"), SqliteClientError::DiversifierIndexOutOfRange => write!(f, "The space of available diversifier indices is exhausted"), - SqliteClientError::AccountUnknown(id) => write!(f, "Account {} does not belong to this wallet.", u32::from(*id)), + SqliteClientError::AccountUnknown(acct_id) => write!(f, "Account {} does not belong to this wallet.", u32::from(*acct_id)), - SqliteClientError::KeyDerivationError(acct_id) => write!(f, "Key derivation failed for account {:?}", acct_id), + SqliteClientError::KeyDerivationError(acct_id) => write!(f, "Key derivation failed for account {}", u32::from(*acct_id)), SqliteClientError::AccountIdDiscontinuity => write!(f, "Wallet account identifiers must be sequential."), SqliteClientError::AccountIdOutOfRange => write!(f, "Wallet account identifiers must be less than 0x7FFFFFFF."), #[cfg(feature = "transparent-inputs")] diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index e8b1c0e293..678ad6f47c 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -165,9 +165,6 @@ pub(crate) fn add_account( key: &UnifiedFullViewingKey, birthday: AccountBirthday, ) -> Result<(), SqliteClientError> { - // Set the wallet birthday, falling back to the chain tip if not specified - let chain_tip = scan_queue_extrema(conn)?.map(|(_, max)| max); - conn.execute( "INSERT INTO accounts (account, ufvk, birthday_height, recover_until_height) VALUES (:account, :ufvk, :birthday_height, :recover_until_height)", @@ -227,9 +224,9 @@ pub(crate) fn add_account( )?; }; - // Rewrite the scan ranges starting from the birthday height so that we'll ensure we + // Rewrite the scan ranges from the birthday height up to the chain tip so that we'll ensure we // re-scan to find any notes that might belong to the newly added account. - if let Some(t) = chain_tip { + if let Some(t) = scan_queue_extrema(conn)?.map(|(_, max)| max) { let rescan_range = birthday.height()..(t + 1); replace_queue_entries::( @@ -240,7 +237,7 @@ pub(crate) fn add_account( ScanPriority::Historic, )) .into_iter(), - true, + true, // force rescan )?; } @@ -1687,7 +1684,6 @@ mod tests { #[cfg(feature = "transparent-inputs")] use { - incrementalmerkletree::frontier::Frontier, secrecy::Secret, zcash_client_backend::{ data_api::WalletWrite, encoding::AddressCodec, wallet::WalletTransparentOutput, @@ -1738,8 +1734,7 @@ mod tests { // Add an account to the wallet let seed = Secret::new([0u8; 32].to_vec()); - let birthday = - AccountBirthday::from_parts(st.sapling_activation_height(), Frontier::empty(), None); + let birthday = AccountBirthday::from_sapling_activation(&st.network()); let (account_id, _usk) = st.wallet_mut().create_account(&seed, birthday).unwrap(); let uaddr = st .wallet() diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 7d1d9ed3ca..18efa866a3 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -543,23 +543,23 @@ pub(crate) fn replace_queue_entries( let mut suggested_stmt = conn .prepare_cached( "SELECT block_range_start, block_range_end, priority - FROM scan_queue - WHERE ( - -- the start is contained within or adjacent to the range - :start >= block_range_start - AND :start <= block_range_end - ) - OR ( - -- the end is contained within or adjacent to the range - :end >= block_range_start - AND :end <= block_range_end - ) - OR ( - -- start..end contains the entire range - block_range_start >= :start - AND block_range_end <= :end - ) - ORDER BY block_range_end", + FROM scan_queue + WHERE ( + -- the start is contained within or adjacent to the range + :start >= block_range_start + AND :start <= block_range_end + ) + OR ( + -- the end is contained within or adjacent to the range + :end >= block_range_start + AND :end <= block_range_end + ) + OR ( + -- start..end contains the entire range + block_range_start >= :start + AND block_range_end <= :end + ) + ORDER BY block_range_end", ) .map_err(E::db_error)?; From 976b0f093153ff8f72aaae00066105c9ce6f2025 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Tue, 5 Sep 2023 15:25:26 -0600 Subject: [PATCH 0139/1122] Check for potential discontinuities when inserting into the `*_tree_shards` table. Closes #924 --- .../src/wallet/commitment_tree.rs | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 01302dca05..6618cae753 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -4,6 +4,7 @@ use std::{ error, fmt, io::{self, Cursor}, marker::PhantomData, + ops::Range, sync::Arc, }; use zcash_client_backend::data_api::chain::CommitmentTreeRoot; @@ -35,6 +36,13 @@ pub enum Error { extant_tree_state: TreeState, extant_marks_removed: Option>, }, + /// Raised when attempting to add shard roots to the database that + /// are discontinuous with the existing roots in the database. + SubtreeDiscontinuity { + attempted_insertion_range: Range, + min: u64, + max: u64, + }, } impl fmt::Display for Error { @@ -54,6 +62,17 @@ impl fmt::Display for Error { checkpoint_id, checkpoint, extant_tree_state, extant_marks_removed ) } + Error::SubtreeDiscontinuity { + attempted_insertion_range, + min, + max, + } => { + write!( + f, + "Attempted to write subtree roots with indicies {:?} which is discontinuous with existing subtree range {:?}", + attempted_insertion_range, *min..(*max+1) + ) + } } } } @@ -64,6 +83,7 @@ impl error::Error for Error { Error::Serialization(e) => Some(e), Error::Query(e) => Some(e), Error::CheckpointConflict { .. } => None, + Error::SubtreeDiscontinuity { .. } => None, } } } @@ -367,6 +387,46 @@ pub(crate) fn last_shard( .transpose() } +/// Returns an error iff the proposed insertion range +/// for the tree shards would create a discontinuity +/// in the database. +fn check_shard_discontinuity( + conn: &rusqlite::Connection, + table_prefix: &'static str, + proposed_insertion_range: Range, +) -> Result<(), Error> { + if let Ok((Some(min), Some(max))) = conn + .query_row( + &format!( + "SELECT MIN(shard_index), MAX(shard_index) FROM {}_tree_shards", + table_prefix + ), + [], + |row| { + let min = row.get::<_, Option>(0)?; + let max = row.get::<_, Option>(1)?; + Ok((min, max)) + }, + ) + .map_err(Error::Query) + { + if !proposed_insertion_range.contains(&min) && !proposed_insertion_range.contains(&max) { + // The proposed insertion range does not overlap with the existing shard indicies. + // This means a discontinuity is introduced if the proposed insertion range's start + // is not `max + 1` _and_ `min` is not the proposed insertion range's end. + if max + 1 != proposed_insertion_range.start && min != proposed_insertion_range.end { + return Err(Error::SubtreeDiscontinuity { + attempted_insertion_range: proposed_insertion_range, + min, + max, + }); + } + } + } + + Ok(()) +} + pub(crate) fn put_shard( conn: &rusqlite::Transaction<'_>, table_prefix: &'static str, @@ -388,6 +448,10 @@ pub(crate) fn put_shard( let mut subtree_data = vec![]; write_shard(&mut subtree_data, subtree.root()).map_err(Error::Serialization)?; + let shard_index = subtree.root_addr().index(); + + check_shard_discontinuity(conn, table_prefix, shard_index..shard_index + 1)?; + let mut stmt_put_shard = conn .prepare_cached(&format!( "INSERT INTO {}_tree_shards (shard_index, root_hash, shard_data) @@ -401,7 +465,7 @@ pub(crate) fn put_shard( stmt_put_shard .execute(named_params![ - ":shard_index": subtree.root_addr().index(), + ":shard_index": shard_index, ":root_hash": subtree_root_hash, ":shard_data": subtree_data ]) @@ -934,6 +998,13 @@ pub(crate) fn put_shard_roots< put_cap(conn, table_prefix, cap_result.subtree.take_root()).map_err(ShardTreeError::Storage)?; + check_shard_discontinuity( + conn, + table_prefix, + start_index..start_index + (roots.len() as u64), + ) + .map_err(ShardTreeError::Storage)?; + for (root, i) in roots.iter().zip(0u64..) { // We want to avoid deserializing the subtree just to annotate its root node, so we simply // cache the downloaded root alongside of any already-persisted subtree. We will update the From f53ea2d7787c59477596b52eac1020d612033c34 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 15 Aug 2023 08:35:07 -0600 Subject: [PATCH 0140/1122] Add `get_wallet_summary` to `WalletRead` The intent of this API is to provide a single API which returns in a single call: * per-account balances, including pending values * wallet sync progress Fixes #865 Fixes #900 --- zcash_client_backend/CHANGELOG.md | 8 +- zcash_client_backend/src/data_api.rs | 188 ++++++++- zcash_client_sqlite/src/lib.rs | 17 +- zcash_client_sqlite/src/testing.rs | 76 +++- zcash_client_sqlite/src/wallet.rs | 373 ++++++++++++++---- zcash_client_sqlite/src/wallet/init.rs | 54 ++- .../src/wallet/init/migrations.rs | 4 + .../v_sapling_shard_unscanned_ranges.rs | 32 +- .../init/migrations/wallet_summaries.rs | 89 +++++ zcash_client_sqlite/src/wallet/sapling.rs | 140 +++---- zcash_client_sqlite/src/wallet/scanning.rs | 23 +- zcash_primitives/CHANGELOG.md | 5 + .../src/transaction/components/amount.rs | 31 ++ 13 files changed, 839 insertions(+), 201 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/wallet_summaries.rs diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 80042ceade..5740dbaea5 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -22,15 +22,19 @@ and this library adheres to Rust's notion of - `impl Eq for zcash_client_backend::zip321::{Payment, TransactionRequest}` - `impl Debug` for `zcash_client_backend::{data_api::wallet::input_selection::Proposal, wallet::ReceivedSaplingNote}` - `zcash_client_backend::data_api`: + - `AccountBalance` - `AccountBirthday` + - `Balance` - `BlockMetadata` - `NoteId` - `NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` + - `Ratio` - `ScannedBlock` - `ShieldedProtocol` - `WalletCommitmentTrees` + - `WalletSummary` - `WalletRead::{chain_height, block_metadata, block_fully_scanned, suggest_scan_ranges, - get_wallet_birthday, get_account_birthday}` + get_wallet_birthday, get_account_birthday, get_wallet_summary}` - `WalletWrite::{put_blocks, update_chain_tip}` - `chain::CommitmentTreeRoot` - `scanning` A new module containing types required for `suggest_scan_ranges` @@ -113,6 +117,8 @@ and this library adheres to Rust's notion of instead to obtain the wallet's view of the chain tip instead, or `suggest_scan_ranges` to obtain information about blocks that need to be scanned. + - `WalletRead::get_balance_at` has been removed. Use `WalletRead::get_wallet_summary` + instead. - `WalletRead::{get_all_nullifiers, get_commitment_tree, get_witnesses}` have been removed without replacement. The utility of these methods is now subsumed by those available from the `WalletCommitmentTrees` trait. diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 2211318e47..df923d756b 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -1,9 +1,11 @@ //! Interfaces for wallet data persistence & low-level wallet utilities. -use std::fmt::Debug; -use std::io; -use std::num::NonZeroU32; -use std::{collections::HashMap, num::TryFromIntError}; +use std::{ + collections::{BTreeMap, HashMap}, + fmt::Debug, + io, + num::{NonZeroU32, TryFromIntError}, +}; use incrementalmerkletree::{frontier::Frontier, Retention}; use secrecy::SecretVec; @@ -15,7 +17,10 @@ use zcash_primitives::{ memo::{Memo, MemoBytes}, sapling::{self, Node, NOTE_COMMITMENT_TREE_DEPTH}, transaction::{ - components::{amount::Amount, OutPoint}, + components::{ + amount::{Amount, NonNegativeAmount}, + OutPoint, + }, Transaction, TxId, }, zip32::{AccountId, ExtendedFullViewingKey}, @@ -46,6 +51,155 @@ pub enum NullifierQuery { All, } +/// Balance information for a value within a single shielded pool in an account. +#[derive(Debug, Clone, Copy)] +pub struct Balance { + /// The value in the account that may currently be spent; it is possible to compute witnesses + /// for all the notes that comprise this value, and all of this value is confirmed to the + /// required confirmation depth. + pub spendable_value: NonNegativeAmount, + + /// The value in the account of shielded change notes that do not yet have sufficient + /// confirmations to be spendable. + pub change_pending_confirmation: NonNegativeAmount, + + /// The value in the account of all remaining received notes that either do not have sufficient + /// confirmations to be spendable, or for which witnesses cannot yet be constructed without + /// additional scanning. + pub value_pending_spendability: NonNegativeAmount, +} + +impl Balance { + /// The [`Balance`] value having zero values for all its fields. + pub const ZERO: Self = Self { + spendable_value: NonNegativeAmount::ZERO, + change_pending_confirmation: NonNegativeAmount::ZERO, + value_pending_spendability: NonNegativeAmount::ZERO, + }; + + /// Returns the total value of funds represented by this [`Balance`]. + pub fn total(&self) -> NonNegativeAmount { + (self.spendable_value + self.change_pending_confirmation + self.value_pending_spendability) + .expect("Balance cannot overflow MAX_MONEY") + } +} + +/// Balance information for a single account. The sum of this struct's fields is the total balance +/// of the wallet. +#[derive(Debug, Clone, Copy)] +pub struct AccountBalance { + /// The value of unspent Sapling outputs belonging to the account. + pub sapling_balance: Balance, + + /// The value of all unspent transparent outputs belonging to the account, irrespective of + /// confirmation depth. + /// + /// Unshielded balances are not subject to confirmation-depth constraints, because the only + /// possible operation on a transparent balance is to shield it, it is possible to create a + /// zero-conf transaction to perform that shielding, and the resulting shielded notes will be + /// subject to normal confirmation rules. + pub unshielded: NonNegativeAmount, +} + +impl AccountBalance { + /// Returns the total value of funds belonging to the account. + pub fn total(&self) -> NonNegativeAmount { + (self.sapling_balance.total() + self.unshielded) + .expect("Account balance cannot overflow MAX_MONEY") + } +} + +/// A polymorphic ratio type, usually used for rational numbers. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Ratio { + numerator: T, + denominator: T, +} + +impl Ratio { + /// Constructs a new Ratio from a numerator and a denominator. + pub fn new(numerator: T, denominator: T) -> Self { + Self { + numerator, + denominator, + } + } + + /// Returns the numerator of the ratio. + pub fn numerator(&self) -> &T { + &self.numerator + } + + /// Returns the denominator of the ratio. + pub fn denominator(&self) -> &T { + &self.denominator + } +} + +/// A type representing the potentially-spendable value of unspent outputs in the wallet. +/// +/// The balances reported using this data structure may overestimate the total spendable value of +/// the wallet, in the case that the spend of a previously received shielded note has not yet been +/// detected by the process of scanning the chain. The balances reported using this data structure +/// can only be certain to be unspent in the case that [`Self::is_synced`] is true, and even in +/// this circumstance it is possible that a newly created transaction could conflict with a +/// not-yet-mined transaction in the mempool. +#[derive(Debug, Clone)] +pub struct WalletSummary { + account_balances: BTreeMap, + chain_tip_height: BlockHeight, + fully_scanned_height: BlockHeight, + sapling_scan_progress: Option>, +} + +impl WalletSummary { + /// Constructs a new [`WalletSummary`] from its constituent parts. + pub fn new( + account_balances: BTreeMap, + chain_tip_height: BlockHeight, + fully_scanned_height: BlockHeight, + sapling_scan_progress: Option>, + ) -> Self { + Self { + account_balances, + chain_tip_height, + fully_scanned_height, + sapling_scan_progress, + } + } + + /// Returns the balances of accounts in the wallet, keyed by account ID. + pub fn account_balances(&self) -> &BTreeMap { + &self.account_balances + } + + /// Returns the height of the current chain tip. + pub fn chain_tip_height(&self) -> BlockHeight { + self.chain_tip_height + } + + /// Returns the height below which all blocks wallet have been scanned, ignoring blocks below + /// the wallet birthday. + pub fn fully_scanned_height(&self) -> BlockHeight { + self.fully_scanned_height + } + + /// Returns the progress of scanning Sapling outputs, in terms of the ratio between notes + /// scanned and the total number of notes added to the chain since the wallet birthday. + /// + /// This ratio should only be used to compute progress percentages, and the numerator and + /// denominator should not be treated as authoritative note counts. Returns `None` if the + /// wallet is unable to determine the size of the note commitment tree. + pub fn sapling_scan_progress(&self) -> Option> { + self.sapling_scan_progress + } + + /// Returns whether or not wallet scanning is complete. + pub fn is_synced(&self) -> bool { + self.chain_tip_height == self.fully_scanned_height + } +} + /// Read-only operations required for light wallet functions. /// /// This trait defines the read-only portion of the storage interface atop which @@ -157,15 +311,12 @@ pub trait WalletRead { extfvk: &ExtendedFullViewingKey, ) -> Result; - /// Returns the wallet balance for an account as of the specified block height. - /// - /// This may be used to obtain a balance that ignores notes that have been received so recently - /// that they are not yet deemed spendable. - fn get_balance_at( + /// Returns the wallet balances and sync status for an account given the specified minimum + /// number of confirmations, or `Ok(None)` if the wallet has no balance data available. + fn get_wallet_summary( &self, - account: AccountId, - anchor_height: BlockHeight, - ) -> Result; + min_confirmations: u32, + ) -> Result, Self::Error>; /// Returns the memo for a note. /// @@ -747,7 +898,7 @@ pub mod testing { use super::{ chain::CommitmentTreeRoot, scanning::ScanRange, AccountBirthday, BlockMetadata, DecryptedTransaction, NoteId, NullifierQuery, ScannedBlock, SentTransaction, - WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, + WalletCommitmentTrees, WalletRead, WalletSummary, WalletWrite, SAPLING_SHARD_HEIGHT, }; pub struct MockWalletDb { @@ -853,12 +1004,11 @@ pub mod testing { Ok(false) } - fn get_balance_at( + fn get_wallet_summary( &self, - _account: AccountId, - _anchor_height: BlockHeight, - ) -> Result { - Ok(Amount::zero()) + _min_confirmations: u32, + ) -> Result, Self::Error> { + Ok(None) } fn get_memo(&self, _id_note: NoteId) -> Result, Self::Error> { diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 318fc16abc..670f5f97bd 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -66,7 +66,7 @@ use zcash_client_backend::{ scanning::{ScanPriority, ScanRange}, AccountBirthday, BlockMetadata, DecryptedTransaction, NoteId, NullifierQuery, PoolType, Recipient, ScannedBlock, SentTransaction, ShieldedProtocol, WalletCommitmentTrees, - WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, + WalletRead, WalletSummary, WalletWrite, SAPLING_SHARD_HEIGHT, }, keys::{UnifiedFullViewingKey, UnifiedSpendingKey}, proto::compact_formats::CompactBlock, @@ -88,7 +88,10 @@ pub mod error; pub mod serialization; pub mod wallet; -use wallet::commitment_tree::{self, put_shard_roots}; +use wallet::{ + commitment_tree::{self, put_shard_roots}, + SubtreeScanProgress, +}; #[cfg(test)] mod testing; @@ -243,12 +246,11 @@ impl, P: consensus::Parameters> WalletRead for W wallet::is_valid_account_extfvk(self.conn.borrow(), &self.params, account, extfvk) } - fn get_balance_at( + fn get_wallet_summary( &self, - account: AccountId, - anchor_height: BlockHeight, - ) -> Result { - wallet::get_balance_at(self.conn.borrow(), account, anchor_height) + min_confirmations: u32, + ) -> Result, Self::Error> { + wallet::get_wallet_summary(self.conn.borrow(), min_confirmations, &SubtreeScanProgress) } fn get_memo(&self, note_id: NoteId) -> Result, Self::Error> { @@ -456,6 +458,7 @@ impl WalletWrite for WalletDb block.block_hash(), block.block_time(), block.metadata().sapling_tree_size(), + block.sapling_commitments().len().try_into().unwrap(), )?; for tx in block.transactions() { diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index d16af5a39a..7fbb46e192 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -25,7 +25,7 @@ use zcash_client_backend::{ input_selection::{GreedyInputSelectorError, InputSelector, Proposal}, propose_transfer, spend, }, - AccountBirthday, WalletWrite, + AccountBirthday, WalletSummary, WalletWrite, }, keys::UnifiedSpendingKey, proto::compact_formats::{ @@ -46,7 +46,10 @@ use zcash_primitives::{ Note, Nullifier, PaymentAddress, }, transaction::{ - components::{amount::BalanceError, Amount}, + components::{ + amount::{BalanceError, NonNegativeAmount}, + Amount, + }, fees::FeeRule, TxId, }, @@ -56,7 +59,10 @@ use zcash_primitives::{ use crate::{ chain::init::init_cache_database, error::SqliteClientError, - wallet::{commitment_tree, init::init_wallet_db, sapling::tests::test_prover}, + wallet::{ + commitment_tree, get_wallet_summary, init::init_wallet_db, sapling::tests::test_prover, + SubtreeScanProgress, + }, AccountId, ReceivedNoteId, WalletDb, }; @@ -65,9 +71,7 @@ use super::BlockDb; #[cfg(feature = "transparent-inputs")] use { zcash_client_backend::data_api::wallet::{propose_shielding, shield_transparent_funds}, - zcash_primitives::{ - legacy::TransparentAddress, transaction::components::amount::NonNegativeAmount, - }, + zcash_primitives::legacy::TransparentAddress, }; #[cfg(feature = "unstable")] @@ -286,6 +290,66 @@ where limit, ) } + + pub(crate) fn get_total_balance(&self, account: AccountId) -> NonNegativeAmount { + get_wallet_summary(&self.wallet().conn, 0, &SubtreeScanProgress) + .unwrap() + .unwrap() + .account_balances() + .get(&account) + .unwrap() + .total() + } + + pub(crate) fn get_spendable_balance( + &self, + account: AccountId, + min_confirmations: u32, + ) -> NonNegativeAmount { + let binding = + get_wallet_summary(&self.wallet().conn, min_confirmations, &SubtreeScanProgress) + .unwrap() + .unwrap(); + let balance = binding.account_balances().get(&account).unwrap(); + + balance.sapling_balance.spendable_value + } + + pub(crate) fn get_pending_shielded_balance( + &self, + account: AccountId, + min_confirmations: u32, + ) -> NonNegativeAmount { + let binding = + get_wallet_summary(&self.wallet().conn, min_confirmations, &SubtreeScanProgress) + .unwrap() + .unwrap(); + let balance = binding.account_balances().get(&account).unwrap(); + + (balance.sapling_balance.value_pending_spendability + + balance.sapling_balance.change_pending_confirmation) + .unwrap() + } + + pub(crate) fn get_pending_change( + &self, + account: AccountId, + min_confirmations: u32, + ) -> NonNegativeAmount { + let binding = + get_wallet_summary(&self.wallet().conn, min_confirmations, &SubtreeScanProgress) + .unwrap() + .unwrap(); + let balance = binding.account_balances().get(&account).unwrap(); + + balance.sapling_balance.change_pending_confirmation + } + + pub(crate) fn get_wallet_summary(&self, min_confirmations: u32) -> WalletSummary { + get_wallet_summary(&self.wallet().conn, min_confirmations, &SubtreeScanProgress) + .unwrap() + .unwrap() + } } impl TestState { diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index e8b1c0e293..41b775102c 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -68,11 +68,13 @@ use incrementalmerkletree::Retention; use rusqlite::{self, named_params, OptionalExtension, ToSql}; use shardtree::ShardTree; use std::cmp; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::convert::TryFrom; use std::io::{self, Cursor}; use std::num::NonZeroU32; use tracing::debug; +use zcash_client_backend::data_api::{AccountBalance, Balance, Ratio, WalletSummary}; +use zcash_primitives::transaction::components::amount::NonNegativeAmount; use zcash_client_backend::data_api::{ scanning::{ScanPriority, ScanRange}, @@ -106,7 +108,7 @@ use crate::{ }; use crate::{SAPLING_TABLES_PREFIX, VERIFY_LOOKAHEAD}; -use self::scanning::replace_queue_entries; +use self::scanning::{parse_priority_code, replace_queue_entries}; #[cfg(feature = "transparent-inputs")] use { @@ -519,28 +521,264 @@ pub(crate) fn get_balance( } } -/// Returns the verified balance for the account at the specified height, -/// This may be used to obtain a balance that ignores notes that have been -/// received so recently that they are not yet deemed spendable. -pub(crate) fn get_balance_at( +pub(crate) trait ScanProgress { + fn sapling_scan_progress( + &self, + conn: &rusqlite::Connection, + birthday_height: BlockHeight, + fully_scanned_height: BlockHeight, + chain_tip_height: BlockHeight, + ) -> Result>, SqliteClientError>; +} + +pub(crate) struct SubtreeScanProgress; + +impl ScanProgress for SubtreeScanProgress { + fn sapling_scan_progress( + &self, + conn: &rusqlite::Connection, + birthday_height: BlockHeight, + fully_scanned_height: BlockHeight, + chain_tip_height: BlockHeight, + ) -> Result>, SqliteClientError> { + if fully_scanned_height == chain_tip_height { + // Compute the total blocks scanned since the wallet birthday + conn.query_row( + "SELECT SUM(sapling_output_count) + FROM blocks + WHERE height >= :birthday_height", + named_params![":birthday_height": u32::from(birthday_height)], + |row| { + let scanned = row.get::<_, Option>(0)?; + Ok(scanned.map(|n| Ratio::new(n, n))) + }, + ) + .map_err(SqliteClientError::from) + } else { + // Compute the number of fully scanned notes directly from the blocks table + let fully_scanned_size = conn.query_row( + "SELECT MAX(sapling_commitment_tree_size) + FROM blocks + WHERE height <= :fully_scanned_height", + named_params![":fully_scanned_height": u32::from(fully_scanned_height)], + |row| row.get::<_, Option>(0), + )?; + + // Compute the total blocks scanned so far above the fully scanned height + let scanned_count = conn.query_row( + "SELECT SUM(sapling_output_count) + FROM blocks + WHERE height > :fully_scanned_height", + named_params![":fully_scanned_height": u32::from(fully_scanned_height)], + |row| row.get::<_, Option>(0), + )?; + + // We don't have complete information on how many outputs will exist in the shard at + // the chain tip without having scanned the chain tip block, so we overestimate by + // computing the maximum possible number of notes directly from the shard indices. + // + // TODO: it would be nice to be able to reliably have the size of the commitment tree + // at the chain tip without having to have scanned that block. + Ok(conn + .query_row( + "SELECT MIN(shard_index), MAX(shard_index) + FROM sapling_tree_shards + WHERE subtree_end_height > :fully_scanned_height + OR subtree_end_height IS NULL", + named_params![":fully_scanned_height": u32::from(fully_scanned_height)], + |row| { + let min_tree_size = row + .get::<_, Option>(0)? + .map(|min| min << SAPLING_SHARD_HEIGHT); + let max_idx = row.get::<_, Option>(1)?; + Ok(fully_scanned_size.or(min_tree_size).zip(max_idx).map( + |(min_tree_size, max)| { + let max_tree_size = (max + 1) << SAPLING_SHARD_HEIGHT; + Ratio::new( + scanned_count.unwrap_or(0), + max_tree_size - min_tree_size, + ) + }, + )) + }, + ) + .optional()? + .flatten()) + } + } +} + +/// Returns the spendable balance for the account at the specified height. +/// +/// This may be used to obtain a balance that ignores notes that have been detected so recently +/// that they are not yet spendable, or for which it is not yet possible to construct witnesses. +pub(crate) fn get_wallet_summary( conn: &rusqlite::Connection, - account: AccountId, - anchor_height: BlockHeight, -) -> Result { - let balance = conn.query_row( - "SELECT SUM(value) FROM sapling_received_notes - INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx - WHERE account = ? AND spent IS NULL AND transactions.block <= ?", - [u32::from(account), u32::from(anchor_height)], - |row| row.get(0).or(Ok(0)), + min_confirmations: u32, + progress: &impl ScanProgress, +) -> Result, SqliteClientError> { + let chain_tip_height = match scan_queue_extrema(conn)? { + Some((_, max)) => max, + None => { + return Ok(None); + } + }; + + let birthday_height = + wallet_birthday(conn)?.expect("If a scan range exists, we know the wallet birthday."); + + let fully_scanned_height = + block_fully_scanned(conn)?.map_or(birthday_height - 1, |m| m.block_height()); + let summary_height = chain_tip_height + 1 - min_confirmations; + + let sapling_scan_progress = progress.sapling_scan_progress( + conn, + birthday_height, + fully_scanned_height, + chain_tip_height, )?; - match Amount::from_i64(balance) { - Ok(amount) if !amount.is_negative() => Ok(amount), - _ => Err(SqliteClientError::CorruptedData( - "Sum of values in sapling_received_notes is out of range".to_string(), - )), + // If the shard containing the anchor is contains any unscanned ranges below the summary + // height, none of our balance is currently spendable. + let any_spendable = conn.query_row( + "SELECT EXISTS( + SELECT 1 FROM v_sapling_shard_unscanned_ranges + WHERE :summary_height + BETWEEN subtree_start_height + AND IFNULL(subtree_end_height, :summary_height) + AND block_range_start <= :summary_height + )", + named_params![":summary_height": u32::from(summary_height)], + |row| row.get::<_, bool>(0).map(|b| !b), + )?; + + let mut stmt_select_notes = conn.prepare_cached( + "SELECT n.account, n.value, n.is_change, scan_state.max_priority, t.block, t.expiry_height + FROM sapling_received_notes n + JOIN transactions t ON t.id_tx = n.tx + LEFT OUTER JOIN v_sapling_shards_scan_state scan_state + ON n.commitment_tree_position >= scan_state.start_position + AND n.commitment_tree_position < scan_state.end_position_exclusive + WHERE n.spent IS NULL + AND ( + t.expiry_height IS NULL + OR t.block IS NOT NULL + OR t.expiry_height >= :summary_height + )", + )?; + + let mut account_balances: BTreeMap = BTreeMap::new(); + let mut rows = + stmt_select_notes.query(named_params![":summary_height": u32::from(summary_height)])?; + while let Some(row) = rows.next()? { + let account = row.get::<_, u32>(0).map(AccountId::from)?; + + let value_raw = row.get::<_, i64>(1)?; + let value = NonNegativeAmount::from_nonnegative_i64(value_raw).map_err(|_| { + SqliteClientError::CorruptedData(format!("Negative received note value: {}", value_raw)) + })?; + + let is_change = row.get::<_, bool>(2)?; + + // If `max_priority` is null, this means that the note is not positioned; the note + // will not be spendable, so we assign the scan priority to `ChainTip` as a priority + // that is greater than `Scanned` + let max_priority_raw = row.get::<_, Option>(3)?; + let max_priority = max_priority_raw.map_or_else( + || Ok(ScanPriority::ChainTip), + |raw| { + parse_priority_code(raw).ok_or_else(|| { + SqliteClientError::CorruptedData(format!( + "Priority code {} not recognized.", + raw + )) + }) + }, + )?; + + let received_height = row + .get::<_, Option>(4) + .map(|opt| opt.map(BlockHeight::from))?; + + let is_spendable = any_spendable + && received_height.iter().any(|h| h <= &summary_height) + && max_priority <= ScanPriority::Scanned; + + let is_pending_change = is_change && received_height.iter().all(|h| h > &summary_height); + + let (spendable_value, change_pending_confirmation, value_pending_spendability) = { + let zero = NonNegativeAmount::ZERO; + if is_spendable { + (value, zero, zero) + } else if is_pending_change { + (zero, value, zero) + } else { + (zero, zero, value) + } + }; + + account_balances + .entry(account) + .and_modify(|bal| { + bal.sapling_balance.spendable_value = (bal.sapling_balance.spendable_value + + spendable_value) + .expect("Spendable value cannot overflow"); + bal.sapling_balance.change_pending_confirmation = + (bal.sapling_balance.change_pending_confirmation + change_pending_confirmation) + .expect("Pending change value cannot overflow"); + bal.sapling_balance.value_pending_spendability = + (bal.sapling_balance.value_pending_spendability + value_pending_spendability) + .expect("Value pending spendability cannot overflow"); + }) + .or_insert(AccountBalance { + sapling_balance: Balance { + spendable_value, + change_pending_confirmation, + value_pending_spendability, + }, + unshielded: NonNegativeAmount::ZERO, + }); + } + + #[cfg(feature = "transparent-inputs")] + { + let mut stmt_transparent_balances = conn.prepare( + "SELECT u.received_by_account, SUM(u.value_zat) + FROM utxos u + LEFT OUTER JOIN transactions tx + ON tx.id_tx = u.spent_in_tx + WHERE u.height <= :max_height + AND tx.block IS NULL + GROUP BY u.received_by_account", + )?; + let mut rows = stmt_transparent_balances + .query(named_params![":max_height": u32::from(summary_height)])?; + + while let Some(row) = rows.next()? { + let account = AccountId::from(row.get::<_, u32>(0)?); + let raw_value = row.get(1)?; + let value = NonNegativeAmount::from_nonnegative_i64(raw_value).map_err(|_| { + SqliteClientError::CorruptedData(format!("Negative UTXO value {:?}", raw_value)) + })?; + + account_balances + .entry(account) + .and_modify(|bal| bal.unshielded = value) + .or_insert(AccountBalance { + sapling_balance: Balance::ZERO, + unshielded: value, + }); + } } + + let summary = WalletSummary::new( + account_balances, + chain_tip_height, + fully_scanned_height, + sapling_scan_progress, + ); + + Ok(Some(summary)) } /// Returns the memo for a received note, if the note is known to the wallet. @@ -828,52 +1066,50 @@ pub(crate) fn block_metadata( pub(crate) fn block_fully_scanned( conn: &rusqlite::Connection, ) -> Result, SqliteClientError> { - // We assume here that the wallet was either initialized via `init_blocks_table`, or - // its birthday is Sapling activation, so the earliest block in the `blocks` table is - // the first fully-scanned block (because it occurs before any wallet activity). - // - // We further assume that the only way we get a contiguous range of block heights in - // the `blocks` table starting with this earliest block, is if all scanning operations - // have been performed on those blocks. This holds because the `blocks` table is only - // altered by `WalletDb::put_blocks` via `put_block`, and the effective combination of - // intra-range linear scanning and the nullifier map ensures that we discover all - // wallet-related information within the contiguous range. - // - // The fully-scanned height is therefore the greatest height in the first contiguous - // range of block rows, which is a combined case of the "gaps and islands" and - // "greatest N per group" SQL query problems. - conn.query_row( - "SELECT height, hash, sapling_commitment_tree_size, sapling_tree - FROM blocks - INNER JOIN ( - WITH contiguous AS ( - SELECT height, ROW_NUMBER() OVER (ORDER BY height) - height AS grp - FROM blocks + if let Some(birthday_height) = wallet_birthday(conn)? { + // We assume that the only way we get a contiguous range of block heights in the `blocks` table + // starting with the birthday block, is if all scanning operations have been performed on those + // blocks. This holds because the `blocks` table is only altered by `WalletDb::put_blocks` via + // `put_block`, and the effective combination of intra-range linear scanning and the nullifier + // map ensures that we discover all wallet-related information within the contiguous range. + // + // The fully-scanned height is therefore the greatest height in the first contiguous range of + // block rows, which is a combined case of the "gaps and islands" and "greatest N per group" + // SQL query problems. + conn.query_row( + "SELECT height, hash, sapling_commitment_tree_size, sapling_tree + FROM blocks + INNER JOIN ( + WITH contiguous AS ( + SELECT height, ROW_NUMBER() OVER (ORDER BY height) - height AS grp + FROM blocks + ) + SELECT MIN(height) AS group_min_height, MAX(height) AS group_max_height + FROM contiguous + GROUP BY grp + HAVING :birthday_height BETWEEN group_min_height AND group_max_height ) - SELECT MAX(height) AS [fully_scanned_height] - FROM contiguous - GROUP BY grp - ORDER BY height - LIMIT 1 + ON height = group_max_height", + named_params![":birthday_height": u32::from(birthday_height)], + |row| { + let height: u32 = row.get(0)?; + let block_hash: Vec = row.get(1)?; + let sapling_tree_size: Option = row.get(2)?; + let sapling_tree: Vec = row.get(3)?; + Ok(( + BlockHeight::from(height), + block_hash, + sapling_tree_size, + sapling_tree, + )) + }, ) - ON height = fully_scanned_height", - [], - |row| { - let height: u32 = row.get(0)?; - let block_hash: Vec = row.get(1)?; - let sapling_tree_size: Option = row.get(2)?; - let sapling_tree: Vec = row.get(3)?; - Ok(( - BlockHeight::from(height), - block_hash, - sapling_tree_size, - sapling_tree, - )) - }, - ) - .optional() - .map_err(SqliteClientError::from) - .and_then(|meta_row| meta_row.map(parse_block_metadata).transpose()) + .optional() + .map_err(SqliteClientError::from) + .and_then(|meta_row| meta_row.map(parse_block_metadata).transpose()) + } else { + Ok(None) + } } /// Returns the block height at which the specified transaction was mined, @@ -1171,6 +1407,7 @@ pub(crate) fn put_block( block_hash: BlockHash, block_time: u32, sapling_commitment_tree_size: u32, + sapling_output_count: u32, ) -> Result<(), SqliteClientError> { let block_hash_data = conn .query_row( @@ -1200,6 +1437,7 @@ pub(crate) fn put_block( hash, time, sapling_commitment_tree_size, + sapling_output_count, sapling_tree ) VALUES ( @@ -1207,19 +1445,22 @@ pub(crate) fn put_block( :hash, :block_time, :sapling_commitment_tree_size, + :sapling_output_count, x'00' ) ON CONFLICT (height) DO UPDATE SET hash = :hash, time = :block_time, - sapling_commitment_tree_size = :sapling_commitment_tree_size", + sapling_commitment_tree_size = :sapling_commitment_tree_size, + sapling_output_count = :sapling_output_count", )?; stmt_upsert_block.execute(named_params![ ":height": u32::from(block_height), ":hash": &block_hash.0[..], ":block_time": block_time, - ":sapling_commitment_tree_size": sapling_commitment_tree_size + ":sapling_commitment_tree_size": sapling_commitment_tree_size, + ":sapling_output_count": sapling_output_count, ])?; Ok(()) diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 98162eebd6..a8ba6a80d2 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -219,7 +219,9 @@ mod tests { time INTEGER NOT NULL, sapling_tree BLOB NOT NULL , sapling_commitment_tree_size INTEGER, - orchard_commitment_tree_size INTEGER)", + orchard_commitment_tree_size INTEGER, + sapling_output_count INTEGER, + orchard_action_count INTEGER)", "CREATE TABLE nullifier_map ( spend_pool INTEGER NOT NULL, nf BLOB NOT NULL, @@ -365,16 +367,15 @@ mod tests { } let expected_views = vec![ - // v_sapling_shard_unscanned_ranges + // v_sapling_shard_scan_ranges format!( - "CREATE VIEW v_sapling_shard_unscanned_ranges AS - WITH wallet_birthday AS (SELECT MIN(birthday_height) AS height FROM accounts) + "CREATE VIEW v_sapling_shard_scan_ranges AS SELECT shard.shard_index, shard.shard_index << 16 AS start_position, (shard.shard_index + 1) << 16 AS end_position_exclusive, IFNULL(prev_shard.subtree_end_height, {}) AS subtree_start_height, - shard.subtree_end_height AS subtree_end_height, + shard.subtree_end_height, shard.contains_marked, scan_queue.block_range_start, scan_queue.block_range_end, @@ -383,18 +384,53 @@ mod tests { LEFT OUTER JOIN sapling_tree_shards prev_shard ON shard.shard_index = prev_shard.shard_index + 1 INNER JOIN scan_queue ON + (scan_queue.block_range_start >= subtree_start_height AND shard.subtree_end_height IS NULL) OR (scan_queue.block_range_start BETWEEN subtree_start_height AND shard.subtree_end_height) OR ((scan_queue.block_range_end - 1) BETWEEN subtree_start_height AND shard.subtree_end_height) OR ( scan_queue.block_range_start <= prev_shard.subtree_end_height AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height - ) - INNER JOIN wallet_birthday - WHERE scan_queue.priority > {} - AND scan_queue.block_range_end > wallet_birthday.height", + )", u32::from(st.network().activation_height(NetworkUpgrade::Sapling).unwrap()), + ), + // v_sapling_shard_unscanned_ranges + format!( + "CREATE VIEW v_sapling_shard_unscanned_ranges AS + WITH wallet_birthday AS (SELECT MIN(birthday_height) AS height FROM accounts) + SELECT + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked, + block_range_start, + block_range_end, + priority + FROM v_sapling_shard_scan_ranges + INNER JOIN wallet_birthday + WHERE priority > {} + AND block_range_end > wallet_birthday.height", priority_code(&ScanPriority::Scanned) ), + // v_sapling_shards_scan_state + "CREATE VIEW v_sapling_shards_scan_state AS + SELECT + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked, + MAX(priority) AS max_priority + FROM v_sapling_shard_scan_ranges + GROUP BY + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked".to_owned(), // v_transactions "CREATE VIEW v_transactions AS WITH diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index f5cd2e2ac2..03381ba2cf 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -12,6 +12,7 @@ mod ufvk_support; mod utxos_table; mod v_sapling_shard_unscanned_ranges; mod v_transactions_net; +mod wallet_summaries; use schemer_rusqlite::RusqliteMigration; use secrecy::SecretVec; @@ -42,6 +43,8 @@ pub(super) fn all_migrations( // add_account_birthdays // | // v_sapling_shard_unscanned_ranges + // | + // wallet_summaries vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -72,5 +75,6 @@ pub(super) fn all_migrations( Box::new(v_sapling_shard_unscanned_ranges::Migration { params: params.clone(), }), + Box::new(wallet_summaries::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs index ea1aa68ce2..7a72942c09 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs @@ -38,14 +38,13 @@ impl RusqliteMigration for Migration

{ fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { transaction.execute_batch( &format!( - "CREATE VIEW v_sapling_shard_unscanned_ranges AS - WITH wallet_birthday AS (SELECT MIN(birthday_height) AS height FROM accounts) + "CREATE VIEW v_sapling_shard_scan_ranges AS SELECT shard.shard_index, shard.shard_index << {} AS start_position, (shard.shard_index + 1) << {} AS end_position_exclusive, IFNULL(prev_shard.subtree_end_height, {}) AS subtree_start_height, - shard.subtree_end_height AS subtree_end_height, + shard.subtree_end_height, shard.contains_marked, scan_queue.block_range_start, scan_queue.block_range_end, @@ -54,22 +53,39 @@ impl RusqliteMigration for Migration

{ LEFT OUTER JOIN sapling_tree_shards prev_shard ON shard.shard_index = prev_shard.shard_index + 1 INNER JOIN scan_queue ON + (scan_queue.block_range_start >= subtree_start_height AND shard.subtree_end_height IS NULL) OR (scan_queue.block_range_start BETWEEN subtree_start_height AND shard.subtree_end_height) OR ((scan_queue.block_range_end - 1) BETWEEN subtree_start_height AND shard.subtree_end_height) OR ( scan_queue.block_range_start <= prev_shard.subtree_end_height AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height - ) - INNER JOIN wallet_birthday - WHERE scan_queue.priority > {} - AND scan_queue.block_range_end > wallet_birthday.height;", + )", SAPLING_SHARD_HEIGHT, SAPLING_SHARD_HEIGHT, u32::from(self.params.activation_height(NetworkUpgrade::Sapling).unwrap()), - priority_code(&ScanPriority::Scanned), ) )?; + transaction.execute_batch(&format!( + "CREATE VIEW v_sapling_shard_unscanned_ranges AS + WITH wallet_birthday AS (SELECT MIN(birthday_height) AS height FROM accounts) + SELECT + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked, + block_range_start, + block_range_end, + priority + FROM v_sapling_shard_scan_ranges + INNER JOIN wallet_birthday + WHERE priority > {} + AND block_range_end > wallet_birthday.height;", + priority_code(&ScanPriority::Scanned), + ))?; + Ok(()) } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/wallet_summaries.rs b/zcash_client_sqlite/src/wallet/init/migrations/wallet_summaries.rs new file mode 100644 index 0000000000..5807bae65c --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/wallet_summaries.rs @@ -0,0 +1,89 @@ +//! This migration adds views and database changes required to provide accurate wallet summaries. + +use std::collections::HashSet; + +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::WalletMigrationError; + +use super::v_sapling_shard_unscanned_ranges; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xc5bf7f71_2297_41ff_89e1_75e07c4e8838); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [v_sapling_shard_unscanned_ranges::MIGRATION_ID] + .into_iter() + .collect() + } + + fn description(&self) -> &'static str { + "Adds views and data required to produce accurate wallet summaries." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + // Add columns to the `blocks` table to track the number of scanned outputs in each block. + // We use the note commitment tree size information that we have in contiguous regions to + // populate this data, but we don't make any attempt to handle the boundary cases because + // we're just using this information for the progress metric, which can be a bit sloppy. + transaction.execute_batch( + "ALTER TABLE blocks ADD COLUMN sapling_output_count INTEGER; + ALTER TABLE blocks ADD COLUMN orchard_action_count INTEGER;", + )?; + + transaction.execute_batch( + // set the number of outputs everywhere that we have sequential Sapling blocks + "CREATE TEMPORARY TABLE block_deltas AS + SELECT + cur.height AS height, + (cur.sapling_commitment_tree_size - prev.sapling_commitment_tree_size) AS sapling_delta, + (cur.orchard_commitment_tree_size - prev.orchard_commitment_tree_size) AS orchard_delta + FROM blocks cur + INNER JOIN blocks prev + ON cur.height = prev.height + 1; + + UPDATE blocks + SET sapling_output_count = block_deltas.sapling_delta, + orchard_action_count = block_deltas.orchard_delta + FROM block_deltas + WHERE block_deltas.height = blocks.height;" + )?; + + transaction.execute_batch( + "CREATE VIEW v_sapling_shards_scan_state AS + SELECT + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked, + MAX(priority) AS max_priority + FROM v_sapling_shard_scan_ranges + GROUP BY + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked;", + )?; + + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + panic!("This migration cannot be reverted."); + } +} diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 099e6d6847..4e1f3702ab 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -437,7 +437,10 @@ pub(crate) mod tests { note_encryption::try_sapling_output_recovery, prover::TxProver, Note, PaymentAddress, }, transaction::{ - components::{amount::BalanceError, Amount}, + components::{ + amount::{BalanceError, NonNegativeAmount}, + Amount, + }, fees::{fixed::FeeRule as FixedFeeRule, zip317::FeeRule as Zip317FeeRule}, Transaction, }, @@ -450,7 +453,7 @@ pub(crate) mod tests { self, error::Error, wallet::input_selection::{GreedyInputSelector, GreedyInputSelectorError}, - AccountBirthday, ShieldedProtocol, WalletRead, + AccountBirthday, Ratio, ShieldedProtocol, WalletRead, }, decrypt_transaction, fees::{fixed, zip317, DustOutputPolicy}, @@ -462,14 +465,14 @@ pub(crate) mod tests { use crate::{ error::SqliteClientError, testing::{AddressType, BlockCache, TestBuilder, TestState}, - wallet::{commitment_tree, get_balance, get_balance_at}, + wallet::{commitment_tree, get_balance}, AccountId, NoteId, ReceivedNoteId, }; #[cfg(feature = "transparent-inputs")] use { zcash_client_backend::{data_api::WalletWrite, wallet::WalletTransparentOutput}, - zcash_primitives::transaction::components::{amount::NonNegativeAmount, OutPoint, TxOut}, + zcash_primitives::transaction::components::{OutPoint, TxOut}, }; pub(crate) fn test_prover() -> impl TxProver { @@ -497,18 +500,13 @@ pub(crate) mod tests { st.scan_cached_blocks(h, 1); // Verified balance matches total balance - let (_, anchor_height) = st - .wallet() - .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) - .unwrap() - .unwrap(); assert_eq!( get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), value ); assert_eq!( - get_balance_at(&st.wallet().conn, AccountId::from(0), anchor_height).unwrap(), - value + st.get_total_balance(account), + NonNegativeAmount::try_from(value).unwrap() ); let to_extsk = ExtendedSpendingKey::master(&[]); @@ -693,47 +691,45 @@ pub(crate) mod tests { .with_test_account(AccountBirthday::from_sapling_activation) .build(); - let (_, usk, _) = st.test_account().unwrap(); + let (account, usk, _) = st.test_account().unwrap(); let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note - let value = Amount::from_u64(50000).unwrap(); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let value = NonNegativeAmount::from_u64(50000).unwrap(); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); st.scan_cached_blocks(h1, 1); // Verified balance matches total balance - let (_, anchor_height) = st - .wallet() - .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) - .unwrap() - .unwrap(); - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - value - ); assert_eq!( - get_balance_at(&st.wallet().conn, AccountId::from(0), anchor_height).unwrap(), - value + get_balance(&st.wallet().conn, account).unwrap(), + value.into() ); + assert_eq!(st.get_total_balance(account), value); + + // Value is considered pending + assert_eq!(st.get_pending_shielded_balance(account, 10), value); + + // Wallet is fully scanned + let summary = st.get_wallet_summary(1); + assert_eq!(summary.sapling_scan_progress(), Some(Ratio::new(1, 1))); // Add more funds to the wallet in a second note - let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); st.scan_cached_blocks(h2, 1); // Verified balance does not include the second note - let (_, anchor_height2) = st - .wallet() - .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) - .unwrap() - .unwrap(); + let total = (value + value).unwrap(); assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - (value + value).unwrap() - ); - assert_eq!( - get_balance_at(&st.wallet().conn, AccountId::from(0), anchor_height2).unwrap(), - value + get_balance(&st.wallet().conn, account).unwrap(), + total.into() ); + assert_eq!(st.get_spendable_balance(account, 2), value); + assert_eq!(st.get_pending_shielded_balance(account, 2), value); + assert_eq!(st.get_total_balance(account), total); + + // Wallet is still fully scanned + let summary = st.get_wallet_summary(1); + assert_eq!(summary.sapling_scan_progress(), Some(Ratio::new(2, 2))); // Spend fails because there are insufficient verified notes let extsk2 = ExtendedSpendingKey::master(&[]); @@ -758,7 +754,7 @@ pub(crate) mod tests { // Mine blocks SAPLING_ACTIVATION_HEIGHT + 2 to 9 until just before the second // note is verified for _ in 2..10 { - st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); } st.scan_cached_blocks(h2 + 1, 8); @@ -781,7 +777,7 @@ pub(crate) mod tests { ); // Mine block 11 so that the second note becomes verified - let (h11, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let (h11, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); st.scan_cached_blocks(h11, 1); // Second spend should now succeed @@ -805,17 +801,14 @@ pub(crate) mod tests { .with_test_account(AccountBirthday::from_sapling_activation) .build(); - let (_, usk, _) = st.test_account().unwrap(); + let (account, usk, _) = st.test_account().unwrap(); let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); st.scan_cached_blocks(h1, 1); - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - value - ); + assert_eq!(get_balance(&st.wallet().conn, account).unwrap(), value); // Send some of the funds to another address let extsk2 = ExtendedSpendingKey::master(&[]); @@ -904,17 +897,14 @@ pub(crate) mod tests { .with_test_account(AccountBirthday::from_sapling_activation) .build(); - let (_, usk, _) = st.test_account().unwrap(); + let (account, usk, _) = st.test_account().unwrap(); let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); st.scan_cached_blocks(h1, 1); - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - value - ); + assert_eq!(get_balance(&st.wallet().conn, account).unwrap(), value); let extsk2 = ExtendedSpendingKey::master(&[]); let addr2 = extsk2.default_address().1; @@ -1005,7 +995,7 @@ pub(crate) mod tests { .with_test_account(AccountBirthday::from_sapling_activation) .build(); - let (_, usk, _) = st.test_account().unwrap(); + let (account, usk, _) = st.test_account().unwrap(); let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note @@ -1014,18 +1004,10 @@ pub(crate) mod tests { st.scan_cached_blocks(h, 1); // Verified balance matches total balance - let (_, anchor_height) = st - .wallet() - .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) - .unwrap() - .unwrap(); - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - value - ); + assert_eq!(get_balance(&st.wallet().conn, account).unwrap(), value); assert_eq!( - get_balance_at(&st.wallet().conn, AccountId::from(0), anchor_height).unwrap(), - value + st.get_total_balance(account), + NonNegativeAmount::try_from(value).unwrap() ); let to = TransparentAddress::PublicKey([7; 20]).into(); @@ -1049,27 +1031,25 @@ pub(crate) mod tests { .with_test_account(AccountBirthday::from_sapling_activation) .build(); - let (_, usk, _) = st.test_account().unwrap(); + let (account, usk, _) = st.test_account().unwrap(); let dfvk = st.test_account_sapling().unwrap(); - // Add funds to the wallet in a single note + // Add funds to the wallet in a single note owned by the internal spending key let value = Amount::from_u64(60000).unwrap(); let (h, _, _) = st.generate_next_block(&dfvk, AddressType::Internal, value); st.scan_cached_blocks(h, 1); // Verified balance matches total balance - let (_, anchor_height) = st - .wallet() - .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) - .unwrap() - .unwrap(); + assert_eq!(get_balance(&st.wallet().conn, account).unwrap(), value); assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - value + st.get_total_balance(account), + NonNegativeAmount::try_from(value).unwrap() ); + + // the balance is considered pending assert_eq!( - get_balance_at(&st.wallet().conn, AccountId::from(0), anchor_height).unwrap(), - value + st.get_pending_shielded_balance(account, 10), + NonNegativeAmount::try_from(value).unwrap() ); let to = TransparentAddress::PublicKey([7; 20]).into(); @@ -1093,7 +1073,7 @@ pub(crate) mod tests { .with_test_account(AccountBirthday::from_sapling_activation) .build(); - let (_, usk, _) = st.test_account().unwrap(); + let (account, usk, _) = st.test_account().unwrap(); let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet @@ -1116,18 +1096,10 @@ pub(crate) mod tests { // Verified balance matches total balance let total = Amount::from_u64(60000).unwrap(); - let (_, anchor_height) = st - .wallet() - .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) - .unwrap() - .unwrap(); - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - total - ); + assert_eq!(get_balance(&st.wallet().conn, account).unwrap(), total); assert_eq!( - get_balance_at(&st.wallet().conn, AccountId::from(0), anchor_height).unwrap(), - total + st.get_total_balance(account), + NonNegativeAmount::try_from(total).unwrap() ); let input_selector = GreedyInputSelector::new( diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 640f6bf9b6..fba6cd4d9c 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -881,7 +881,8 @@ mod tests { use zcash_client_backend::data_api::{ chain::CommitmentTreeRoot, scanning::{ScanPriority, ScanRange}, - AccountBirthday, WalletCommitmentTrees, WalletRead, WalletWrite, + AccountBirthday, Ratio, WalletCommitmentTrees, WalletRead, WalletWrite, + SAPLING_SHARD_HEIGHT, }; use zcash_primitives::{ block::BlockHash, @@ -1689,6 +1690,10 @@ mod tests { ) .unwrap(); + // We have scan ranges and a subtree, but have scanned no blocks. + let summary = st.get_wallet_summary(1); + assert_eq!(summary.sapling_scan_progress(), None); + // Set up prior chain state. This simulates us having imported a wallet // with a birthday 520 blocks below the chain tip. st.wallet_mut().update_chain_tip(prior_tip).unwrap(); @@ -1723,6 +1728,14 @@ mod tests { ); st.scan_cached_blocks(max_scanned, 1); + // We have scanned a block, so we now have a starting tree position, 500 blocks above the + // wallet birthday but before the end of the shard. + let summary = st.get_wallet_summary(1); + assert_eq!( + summary.sapling_scan_progress(), + Some(Ratio::new(1, 0x1 << SAPLING_SHARD_HEIGHT)) + ); + // Now simulate shutting down, and then restarting 70 blocks later, after a shard // has been completed. let last_shard_start = prior_tip + 50; @@ -1758,6 +1771,14 @@ mod tests { let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); assert_eq!(actual, expected); + + // We've crossed a subtree boundary, and so still only have one scanned note but have two + // shards worth of notes to scan. + let summary = st.get_wallet_summary(1); + assert_eq!( + summary.sapling_scan_progress(), + Some(Ratio::new(1, 0x1 << (SAPLING_SHARD_HEIGHT + 1))) + ); } #[test] diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index d5855b66a9..ee6d3861bc 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -16,6 +16,11 @@ and this library adheres to Rust's notion of - `impl HashSer for String` is provided under the `test-dependencies` feature flag. This is a test-only impl; the identity leaf value is `_` and the combining operation is concatenation. +- `zcash_primitives::transaction::components::amount::NonNegativeAmount::ZERO` +- Additional trait implementations for `NonNegativeAmount`: + - `TryFrom for NonNegativeAmount` + - `Add for NonNegativeAmount` + - `Add for Option` ### Changed - `zcash_primitives::transaction`: diff --git a/zcash_primitives/src/transaction/components/amount.rs b/zcash_primitives/src/transaction/components/amount.rs index b4e8c4f1e5..fc7e53a6e4 100644 --- a/zcash_primitives/src/transaction/components/amount.rs +++ b/zcash_primitives/src/transaction/components/amount.rs @@ -238,6 +238,9 @@ impl TryFrom for Amount { pub struct NonNegativeAmount(Amount); impl NonNegativeAmount { + /// Returns the identity `NonNegativeAmount` + pub const ZERO: Self = NonNegativeAmount(Amount(0)); + /// Creates a NonNegativeAmount from a u64. /// /// Returns an error if the amount is outside the range `{0..MAX_MONEY}`. @@ -259,6 +262,34 @@ impl From for Amount { } } +impl TryFrom for NonNegativeAmount { + type Error = (); + + fn try_from(value: Amount) -> Result { + if value.is_negative() { + Err(()) + } else { + Ok(NonNegativeAmount(value)) + } + } +} + +impl Add for NonNegativeAmount { + type Output = Option; + + fn add(self, rhs: NonNegativeAmount) -> Option { + (self.0 + rhs.0).map(NonNegativeAmount) + } +} + +impl Add for Option { + type Output = Self; + + fn add(self, rhs: NonNegativeAmount) -> Option { + self.and_then(|lhs| lhs + rhs) + } +} + /// A type for balance violations in amount addition and subtraction /// (overflow and underflow of allowed ranges) #[derive(Copy, Clone, Debug, PartialEq, Eq)] From 7abd1324dedca54f69873d3d1886f768b776e55b Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 5 Sep 2023 11:58:24 -0600 Subject: [PATCH 0141/1122] Apply suggestions from code review Co-authored-by: Daira Emma Hopwood Co-authored-by: Jack Grigg --- zcash_client_backend/src/data_api.rs | 24 ++-- zcash_client_sqlite/src/chain.rs | 115 ++++++++---------- zcash_client_sqlite/src/testing.rs | 115 +++++++++--------- zcash_client_sqlite/src/wallet.rs | 70 +++-------- .../init/migrations/wallet_summaries.rs | 2 +- zcash_client_sqlite/src/wallet/sapling.rs | 61 ++++------ zcash_client_sqlite/src/wallet/scanning.rs | 6 +- .../src/transaction/components/amount.rs | 16 +++ 8 files changed, 178 insertions(+), 231 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index df923d756b..d42b0da95f 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -51,8 +51,8 @@ pub enum NullifierQuery { All, } -/// Balance information for a value within a single shielded pool in an account. -#[derive(Debug, Clone, Copy)] +/// Balance information for a value within a single pool in an account. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Balance { /// The value in the account that may currently be spent; it is possible to compute witnesses /// for all the notes that comprise this value, and all of this value is confirmed to the @@ -86,7 +86,7 @@ impl Balance { /// Balance information for a single account. The sum of this struct's fields is the total balance /// of the wallet. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct AccountBalance { /// The value of unspent Sapling outputs belonging to the account. pub sapling_balance: Balance, @@ -144,12 +144,12 @@ impl Ratio { /// can only be certain to be unspent in the case that [`Self::is_synced`] is true, and even in /// this circumstance it is possible that a newly created transaction could conflict with a /// not-yet-mined transaction in the mempool. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct WalletSummary { account_balances: BTreeMap, chain_tip_height: BlockHeight, fully_scanned_height: BlockHeight, - sapling_scan_progress: Option>, + scan_progress: Option>, } impl WalletSummary { @@ -158,13 +158,13 @@ impl WalletSummary { account_balances: BTreeMap, chain_tip_height: BlockHeight, fully_scanned_height: BlockHeight, - sapling_scan_progress: Option>, + scan_progress: Option>, ) -> Self { Self { account_balances, chain_tip_height, fully_scanned_height, - sapling_scan_progress, + scan_progress, } } @@ -178,20 +178,20 @@ impl WalletSummary { self.chain_tip_height } - /// Returns the height below which all blocks wallet have been scanned, ignoring blocks below - /// the wallet birthday. + /// Returns the height below which all blocks have been scanned by the wallet, ignoring blocks + /// below the wallet birthday. pub fn fully_scanned_height(&self) -> BlockHeight { self.fully_scanned_height } - /// Returns the progress of scanning Sapling outputs, in terms of the ratio between notes + /// Returns the progress of scanning shielded outputs, in terms of the ratio between notes /// scanned and the total number of notes added to the chain since the wallet birthday. /// /// This ratio should only be used to compute progress percentages, and the numerator and /// denominator should not be treated as authoritative note counts. Returns `None` if the /// wallet is unable to determine the size of the note commitment tree. - pub fn sapling_scan_progress(&self) -> Option> { - self.sapling_scan_progress + pub fn scan_progress(&self) -> Option> { + self.scan_progress } /// Returns whether or not wallet scanning is complete. diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index a69bcce221..950d2e2b30 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -326,7 +326,10 @@ mod tests { use zcash_primitives::{ block::BlockHash, - transaction::{components::Amount, fees::zip317::FeeRule}, + transaction::{ + components::{amount::NonNegativeAmount, Amount}, + fees::zip317::FeeRule, + }, zip32::ExtendedSpendingKey, }; @@ -344,7 +347,7 @@ mod tests { use crate::{ testing::{AddressType, TestBuilder}, - wallet::{get_balance, truncate_to_height}, + wallet::truncate_to_height, AccountId, }; @@ -441,24 +444,21 @@ mod tests { let dfvk = st.test_account_sapling().unwrap(); - // Account balance should be zero - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - Amount::zero() - ); + // Wallet summary is not yet available + assert_eq!(st.get_wallet_summary(0), None); // Create fake CompactBlocks sending value to the address - let value = Amount::from_u64(5).unwrap(); - let value2 = Amount::from_u64(7).unwrap(); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); + let value = NonNegativeAmount::from_u64(5).unwrap(); + let value2 = NonNegativeAmount::from_u64(7).unwrap(); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2.into()); // Scan the cache st.scan_cached_blocks(h, 2); // Account balance should reflect both received notes assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), + st.get_total_balance(AccountId::from(0)), (value + value2).unwrap() ); @@ -469,7 +469,7 @@ mod tests { // Account balance should be unaltered assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), + st.get_total_balance(AccountId::from(0)), (value + value2).unwrap() ); @@ -479,17 +479,14 @@ mod tests { .unwrap(); // Account balance should only contain the first received note - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - value - ); + assert_eq!(st.get_total_balance(AccountId::from(0)), value); // Scan the cache again st.scan_cached_blocks(h, 2); // Account balance should again reflect both received notes assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), + st.get_total_balance(AccountId::from(0)), (value + value2).unwrap() ); } @@ -505,17 +502,14 @@ mod tests { let dfvk = st.test_account_sapling().unwrap(); // Create a block with height SAPLING_ACTIVATION_HEIGHT - let value = Amount::from_u64(50000).unwrap(); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let value = NonNegativeAmount::from_u64(50000).unwrap(); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); st.scan_cached_blocks(h1, 1); - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - value - ); + assert_eq!(st.get_total_balance(AccountId::from(0)), value); // Create blocks to reach SAPLING_ACTIVATION_HEIGHT + 2 - let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - let (h3, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + let (h3, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); // Scan the later block first st.scan_cached_blocks(h3, 1); @@ -523,8 +517,8 @@ mod tests { // Now scan the block of height SAPLING_ACTIVATION_HEIGHT + 1 st.scan_cached_blocks(h2, 1); assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - Amount::from_u64(150_000).unwrap() + st.get_total_balance(AccountId::from(0)), + NonNegativeAmount::from_u64(150_000).unwrap() ); // We can spend the received notes @@ -562,35 +556,29 @@ mod tests { let dfvk = st.test_account_sapling().unwrap(); - // Account balance should be zero - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - Amount::zero() - ); + // Wallet summary is not yet available + assert_eq!(st.get_wallet_summary(0), None); // Create a fake CompactBlock sending value to the address - let value = Amount::from_u64(5).unwrap(); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let value = NonNegativeAmount::from_u64(5).unwrap(); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); // Scan the cache st.scan_cached_blocks(h1, 1); // Account balance should reflect the received note - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - value - ); + assert_eq!(st.get_total_balance(AccountId::from(0)), value); // Create a second fake CompactBlock sending more value to the address - let value2 = Amount::from_u64(7).unwrap(); - let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); + let value2 = NonNegativeAmount::from_u64(7).unwrap(); + let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2.into()); // Scan the cache again st.scan_cached_blocks(h2, 1); // Account balance should reflect both received notes assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), + st.get_total_balance(AccountId::from(0)), (value + value2).unwrap() ); } @@ -603,38 +591,33 @@ mod tests { .build(); let dfvk = st.test_account_sapling().unwrap(); - // Account balance should be zero - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - Amount::zero() - ); + // Wallet summary is not yet available + assert_eq!(st.get_wallet_summary(0), None); // Create a fake CompactBlock sending value to the address - let value = Amount::from_u64(5).unwrap(); + let value = NonNegativeAmount::from_u64(5).unwrap(); let (received_height, _, nf) = - st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); // Scan the cache st.scan_cached_blocks(received_height, 1); // Account balance should reflect the received note - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - value - ); + assert_eq!(st.get_total_balance(AccountId::from(0)), value); // Create a second fake CompactBlock spending value from the address let extsk2 = ExtendedSpendingKey::master(&[0]); let to2 = extsk2.default_address().1; - let value2 = Amount::from_u64(2).unwrap(); - let (spent_height, _) = st.generate_next_block_spending(&dfvk, (nf, value), to2, value2); + let value2 = NonNegativeAmount::from_u64(2).unwrap(); + let (spent_height, _) = + st.generate_next_block_spending(&dfvk, (nf, value.into()), to2, value2.into()); // Scan the cache again st.scan_cached_blocks(spent_height, 1); // Account balance should equal the change assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), + st.get_total_balance(AccountId::from(0)), (value - value2).unwrap() ); } @@ -648,29 +631,27 @@ mod tests { let dfvk = st.test_account_sapling().unwrap(); - // Account balance should be zero - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - Amount::zero() - ); + // Wallet summary is not yet available + assert_eq!(st.get_wallet_summary(0), None); // Create a fake CompactBlock sending value to the address - let value = Amount::from_u64(5).unwrap(); + let value = NonNegativeAmount::from_u64(5).unwrap(); let (received_height, _, nf) = - st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); // Create a second fake CompactBlock spending value from the address let extsk2 = ExtendedSpendingKey::master(&[0]); let to2 = extsk2.default_address().1; - let value2 = Amount::from_u64(2).unwrap(); - let (spent_height, _) = st.generate_next_block_spending(&dfvk, (nf, value), to2, value2); + let value2 = NonNegativeAmount::from_u64(2).unwrap(); + let (spent_height, _) = + st.generate_next_block_spending(&dfvk, (nf, value.into()), to2, value2.into()); // Scan the spending block first. st.scan_cached_blocks(spent_height, 1); // Account balance should equal the change assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), + st.get_total_balance(AccountId::from(0)), (value - value2).unwrap() ); @@ -679,7 +660,7 @@ mod tests { // Account balance should be the same. assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), + st.get_total_balance(AccountId::from(0)), (value - value2).unwrap() ); } diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 7fbb46e192..0034ff4bbb 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -14,6 +14,7 @@ use tempfile::NamedTempFile; #[cfg(feature = "unstable")] use tempfile::TempDir; +use zcash_client_backend::data_api::AccountBalance; #[allow(deprecated)] use zcash_client_backend::{ address::RecipientAddress, @@ -290,66 +291,6 @@ where limit, ) } - - pub(crate) fn get_total_balance(&self, account: AccountId) -> NonNegativeAmount { - get_wallet_summary(&self.wallet().conn, 0, &SubtreeScanProgress) - .unwrap() - .unwrap() - .account_balances() - .get(&account) - .unwrap() - .total() - } - - pub(crate) fn get_spendable_balance( - &self, - account: AccountId, - min_confirmations: u32, - ) -> NonNegativeAmount { - let binding = - get_wallet_summary(&self.wallet().conn, min_confirmations, &SubtreeScanProgress) - .unwrap() - .unwrap(); - let balance = binding.account_balances().get(&account).unwrap(); - - balance.sapling_balance.spendable_value - } - - pub(crate) fn get_pending_shielded_balance( - &self, - account: AccountId, - min_confirmations: u32, - ) -> NonNegativeAmount { - let binding = - get_wallet_summary(&self.wallet().conn, min_confirmations, &SubtreeScanProgress) - .unwrap() - .unwrap(); - let balance = binding.account_balances().get(&account).unwrap(); - - (balance.sapling_balance.value_pending_spendability - + balance.sapling_balance.change_pending_confirmation) - .unwrap() - } - - pub(crate) fn get_pending_change( - &self, - account: AccountId, - min_confirmations: u32, - ) -> NonNegativeAmount { - let binding = - get_wallet_summary(&self.wallet().conn, min_confirmations, &SubtreeScanProgress) - .unwrap() - .unwrap(); - let balance = binding.account_balances().get(&account).unwrap(); - - balance.sapling_balance.change_pending_confirmation - } - - pub(crate) fn get_wallet_summary(&self, min_confirmations: u32) -> WalletSummary { - get_wallet_summary(&self.wallet().conn, min_confirmations, &SubtreeScanProgress) - .unwrap() - .unwrap() - } } impl TestState { @@ -594,6 +535,60 @@ impl TestState { min_confirmations, ) } + + fn with_account_balance T>( + &self, + account: AccountId, + min_confirmations: u32, + f: F, + ) -> T { + let binding = + get_wallet_summary(&self.wallet().conn, min_confirmations, &SubtreeScanProgress) + .unwrap() + .unwrap(); + + f(binding.account_balances().get(&account).unwrap()) + } + + pub(crate) fn get_total_balance(&self, account: AccountId) -> NonNegativeAmount { + self.with_account_balance(account, 0, |balance| balance.total()) + } + + pub(crate) fn get_spendable_balance( + &self, + account: AccountId, + min_confirmations: u32, + ) -> NonNegativeAmount { + self.with_account_balance(account, min_confirmations, |balance| { + balance.sapling_balance.spendable_value + }) + } + + pub(crate) fn get_pending_shielded_balance( + &self, + account: AccountId, + min_confirmations: u32, + ) -> NonNegativeAmount { + self.with_account_balance(account, min_confirmations, |balance| { + balance.sapling_balance.value_pending_spendability + + balance.sapling_balance.change_pending_confirmation + }) + .unwrap() + } + + pub(crate) fn get_pending_change( + &self, + account: AccountId, + min_confirmations: u32, + ) -> NonNegativeAmount { + self.with_account_balance(account, min_confirmations, |balance| { + balance.sapling_balance.change_pending_confirmation + }) + } + + pub(crate) fn get_wallet_summary(&self, min_confirmations: u32) -> Option { + get_wallet_summary(&self.wallet().conn, min_confirmations, &SubtreeScanProgress).unwrap() + } } #[allow(dead_code)] diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 41b775102c..d929bbb462 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -493,34 +493,6 @@ pub(crate) fn is_valid_account_extfvk( }) } -/// Returns the balance for the account, including all mined unspent notes that we know -/// about. -/// -/// WARNING: This balance is potentially unreliable, as mined notes may become unmined due -/// to chain reorgs. You should generally not show this balance to users without some -/// caveat. Use [`get_balance_at`] where you need a more reliable indication of the -/// wallet balance. -#[cfg(test)] -pub(crate) fn get_balance( - conn: &rusqlite::Connection, - account: AccountId, -) -> Result { - let balance = conn.query_row( - "SELECT SUM(value) FROM sapling_received_notes - INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx - WHERE account = ? AND spent IS NULL AND transactions.block IS NOT NULL", - [u32::from(account)], - |row| row.get(0).or(Ok(0)), - )?; - - match Amount::from_i64(balance) { - Ok(amount) if !amount.is_negative() => Ok(amount), - _ => Err(SqliteClientError::CorruptedData( - "Sum of values in sapling_received_notes is out of range".to_string(), - )), - } -} - pub(crate) trait ScanProgress { fn sapling_scan_progress( &self, @@ -629,7 +601,7 @@ pub(crate) fn get_wallet_summary( let fully_scanned_height = block_fully_scanned(conn)?.map_or(birthday_height - 1, |m| m.block_height()); - let summary_height = chain_tip_height + 1 - min_confirmations; + let summary_height = (chain_tip_height + 1).saturating_sub(std::cmp::max(min_confirmations, 1)); let sapling_scan_progress = progress.sapling_scan_progress( conn, @@ -638,10 +610,10 @@ pub(crate) fn get_wallet_summary( chain_tip_height, )?; - // If the shard containing the anchor is contains any unscanned ranges below the summary - // height, none of our balance is currently spendable. + // If the shard containing the summary height contains any unscanned ranges that start below or + // including that height, none of our balance is currently spendable. let any_spendable = conn.query_row( - "SELECT EXISTS( + "SELECT NOT EXISTS( SELECT 1 FROM v_sapling_shard_unscanned_ranges WHERE :summary_height BETWEEN subtree_start_height @@ -649,11 +621,11 @@ pub(crate) fn get_wallet_summary( AND block_range_start <= :summary_height )", named_params![":summary_height": u32::from(summary_height)], - |row| row.get::<_, bool>(0).map(|b| !b), + |row| row.get::<_, bool>(0), )?; let mut stmt_select_notes = conn.prepare_cached( - "SELECT n.account, n.value, n.is_change, scan_state.max_priority, t.block, t.expiry_height + "SELECT n.account, n.value, n.is_change, scan_state.max_priority, t.block FROM sapling_received_notes n JOIN transactions t ON t.id_tx = n.tx LEFT OUTER JOIN v_sapling_shards_scan_state scan_state @@ -696,9 +668,7 @@ pub(crate) fn get_wallet_summary( }, )?; - let received_height = row - .get::<_, Option>(4) - .map(|opt| opt.map(BlockHeight::from))?; + let received_height = row.get::<_, Option>(4)?.map(BlockHeight::from); let is_spendable = any_spendable && received_height.iter().any(|h| h <= &summary_height) @@ -763,7 +733,10 @@ pub(crate) fn get_wallet_summary( account_balances .entry(account) - .and_modify(|bal| bal.unshielded = value) + .and_modify(|bal| { + bal.unshielded = + (bal.unshielded + value).expect("Unshielded value cannot overflow") + }) .or_insert(AccountBalance { sapling_balance: Balance::ZERO, unshielded: value, @@ -1924,8 +1897,6 @@ mod tests { use crate::{testing::TestBuilder, AccountId}; - use super::get_balance; - #[cfg(feature = "transparent-inputs")] use { incrementalmerkletree::frontier::Frontier, @@ -1945,11 +1916,8 @@ mod tests { .with_test_account(AccountBirthday::from_sapling_activation) .build(); - // The account should be empty - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - Amount::zero() - ); + // The account should have no summary information + assert_eq!(st.get_wallet_summary(0), None); // We can't get an anchor height, as we have not scanned any blocks. assert_eq!( @@ -1959,15 +1927,17 @@ mod tests { None ); - // An invalid account has zero balance + // The default address is set for the test account + assert_matches!( + st.wallet().get_current_address(AccountId::from(0)), + Ok(Some(_)) + ); + + // No default address is set for an un-initialized account assert_matches!( st.wallet().get_current_address(AccountId::from(1)), Ok(None) ); - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - Amount::zero() - ); } #[test] diff --git a/zcash_client_sqlite/src/wallet/init/migrations/wallet_summaries.rs b/zcash_client_sqlite/src/wallet/init/migrations/wallet_summaries.rs index 5807bae65c..2be9ca15ab 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/wallet_summaries.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/wallet_summaries.rs @@ -43,7 +43,7 @@ impl RusqliteMigration for Migration { )?; transaction.execute_batch( - // set the number of outputs everywhere that we have sequential Sapling blocks + // set the number of outputs everywhere that we have sequential blocks "CREATE TEMPORARY TABLE block_deltas AS SELECT cur.height AS height, diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 4e1f3702ab..d2a5271c11 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -465,7 +465,7 @@ pub(crate) mod tests { use crate::{ error::SqliteClientError, testing::{AddressType, BlockCache, TestBuilder, TestState}, - wallet::{commitment_tree, get_balance}, + wallet::commitment_tree, AccountId, NoteId, ReceivedNoteId, }; @@ -495,19 +495,12 @@ pub(crate) mod tests { let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note - let value = Amount::from_u64(60000).unwrap(); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let value = NonNegativeAmount::from_u64(60000).unwrap(); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); st.scan_cached_blocks(h, 1); // Verified balance matches total balance - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - value - ); - assert_eq!( - st.get_total_balance(account), - NonNegativeAmount::try_from(value).unwrap() - ); + assert_eq!(st.get_total_balance(account), value); let to_extsk = ExtendedSpendingKey::master(&[]); let to: RecipientAddress = to_extsk.default_address().1.into(); @@ -664,11 +657,8 @@ pub(crate) mod tests { let dfvk = st.test_account_sapling().unwrap(); let to = dfvk.default_address().1.into(); - // Account balance should be zero - assert_eq!( - get_balance(&st.wallet().conn, AccountId::from(0)).unwrap(), - Amount::zero() - ); + // Wallet summary is not yet available + assert_eq!(st.get_wallet_summary(0), None); // We cannot do anything if we aren't synchronised assert_matches!( @@ -700,10 +690,6 @@ pub(crate) mod tests { st.scan_cached_blocks(h1, 1); // Verified balance matches total balance - assert_eq!( - get_balance(&st.wallet().conn, account).unwrap(), - value.into() - ); assert_eq!(st.get_total_balance(account), value); // Value is considered pending @@ -711,7 +697,10 @@ pub(crate) mod tests { // Wallet is fully scanned let summary = st.get_wallet_summary(1); - assert_eq!(summary.sapling_scan_progress(), Some(Ratio::new(1, 1))); + assert_eq!( + summary.and_then(|s| s.scan_progress()), + Some(Ratio::new(1, 1)) + ); // Add more funds to the wallet in a second note let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); @@ -719,17 +708,16 @@ pub(crate) mod tests { // Verified balance does not include the second note let total = (value + value).unwrap(); - assert_eq!( - get_balance(&st.wallet().conn, account).unwrap(), - total.into() - ); assert_eq!(st.get_spendable_balance(account, 2), value); assert_eq!(st.get_pending_shielded_balance(account, 2), value); assert_eq!(st.get_total_balance(account), total); // Wallet is still fully scanned let summary = st.get_wallet_summary(1); - assert_eq!(summary.sapling_scan_progress(), Some(Ratio::new(2, 2))); + assert_eq!( + summary.and_then(|s| s.scan_progress()), + Some(Ratio::new(2, 2)) + ); // Spend fails because there are insufficient verified notes let extsk2 = ExtendedSpendingKey::master(&[]); @@ -805,10 +793,10 @@ pub(crate) mod tests { let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note - let value = Amount::from_u64(50000).unwrap(); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let value = NonNegativeAmount::from_u64(50000).unwrap(); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); st.scan_cached_blocks(h1, 1); - assert_eq!(get_balance(&st.wallet().conn, account).unwrap(), value); + assert_eq!(st.get_total_balance(account), value); // Send some of the funds to another address let extsk2 = ExtendedSpendingKey::master(&[]); @@ -848,7 +836,7 @@ pub(crate) mod tests { st.generate_next_block( &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, - value, + value.into(), ); } st.scan_cached_blocks(h1 + 1, 41); @@ -874,7 +862,7 @@ pub(crate) mod tests { let (h43, _, _) = st.generate_next_block( &ExtendedSpendingKey::master(&[42]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, - value, + value.into(), ); st.scan_cached_blocks(h43, 1); @@ -901,10 +889,10 @@ pub(crate) mod tests { let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note - let value = Amount::from_u64(50000).unwrap(); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let value = NonNegativeAmount::from_u64(50000).unwrap(); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); st.scan_cached_blocks(h1, 1); - assert_eq!(get_balance(&st.wallet().conn, account).unwrap(), value); + assert_eq!(st.get_total_balance(account), value); let extsk2 = ExtendedSpendingKey::master(&[]); let addr2 = extsk2.default_address().1; @@ -975,7 +963,7 @@ pub(crate) mod tests { st.generate_next_block( &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, - value, + value.into(), ); } st.scan_cached_blocks(h1 + 1, 42); @@ -1004,7 +992,6 @@ pub(crate) mod tests { st.scan_cached_blocks(h, 1); // Verified balance matches total balance - assert_eq!(get_balance(&st.wallet().conn, account).unwrap(), value); assert_eq!( st.get_total_balance(account), NonNegativeAmount::try_from(value).unwrap() @@ -1040,7 +1027,6 @@ pub(crate) mod tests { st.scan_cached_blocks(h, 1); // Verified balance matches total balance - assert_eq!(get_balance(&st.wallet().conn, account).unwrap(), value); assert_eq!( st.get_total_balance(account), NonNegativeAmount::try_from(value).unwrap() @@ -1096,7 +1082,6 @@ pub(crate) mod tests { // Verified balance matches total balance let total = Amount::from_u64(60000).unwrap(); - assert_eq!(get_balance(&st.wallet().conn, account).unwrap(), total); assert_eq!( st.get_total_balance(account), NonNegativeAmount::try_from(total).unwrap() diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index fba6cd4d9c..47066e1627 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -1692,7 +1692,7 @@ mod tests { // We have scan ranges and a subtree, but have scanned no blocks. let summary = st.get_wallet_summary(1); - assert_eq!(summary.sapling_scan_progress(), None); + assert_eq!(summary.and_then(|s| s.scan_progress()), None); // Set up prior chain state. This simulates us having imported a wallet // with a birthday 520 blocks below the chain tip. @@ -1732,7 +1732,7 @@ mod tests { // wallet birthday but before the end of the shard. let summary = st.get_wallet_summary(1); assert_eq!( - summary.sapling_scan_progress(), + summary.and_then(|s| s.scan_progress()), Some(Ratio::new(1, 0x1 << SAPLING_SHARD_HEIGHT)) ); @@ -1776,7 +1776,7 @@ mod tests { // shards worth of notes to scan. let summary = st.get_wallet_summary(1); assert_eq!( - summary.sapling_scan_progress(), + summary.and_then(|s| s.scan_progress()), Some(Ratio::new(1, 0x1 << (SAPLING_SHARD_HEIGHT + 1))) ); } diff --git a/zcash_primitives/src/transaction/components/amount.rs b/zcash_primitives/src/transaction/components/amount.rs index fc7e53a6e4..dbb3e85d7c 100644 --- a/zcash_primitives/src/transaction/components/amount.rs +++ b/zcash_primitives/src/transaction/components/amount.rs @@ -290,6 +290,22 @@ impl Add for Option { } } +impl Sub for NonNegativeAmount { + type Output = Option; + + fn sub(self, rhs: NonNegativeAmount) -> Option { + (self.0 - rhs.0).and_then(|amt| NonNegativeAmount::try_from(amt).ok()) + } +} + +impl Sub for Option { + type Output = Self; + + fn sub(self, rhs: NonNegativeAmount) -> Option { + self.and_then(|lhs| lhs - rhs) + } +} + /// A type for balance violations in amount addition and subtraction /// (overflow and underflow of allowed ranges) #[derive(Copy, Clone, Debug, PartialEq, Eq)] From 6cbb107c71124b78cde1a19dc52173fb447045b1 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 5 Sep 2023 16:32:28 -0600 Subject: [PATCH 0142/1122] zcash_client_sqlite: allow zero-conf transactions in unshielded balance. --- zcash_client_sqlite/src/wallet.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index d929bbb462..cc52f40ab9 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -712,6 +712,7 @@ pub(crate) fn get_wallet_summary( #[cfg(feature = "transparent-inputs")] { + let zero_conf_height = (chain_tip_height + 1).saturating_sub(min_confirmations); let mut stmt_transparent_balances = conn.prepare( "SELECT u.received_by_account, SUM(u.value_zat) FROM utxos u @@ -722,7 +723,7 @@ pub(crate) fn get_wallet_summary( GROUP BY u.received_by_account", )?; let mut rows = stmt_transparent_balances - .query(named_params![":max_height": u32::from(summary_height)])?; + .query(named_params![":max_height": u32::from(zero_conf_height)])?; while let Some(row) = rows.next()? { let account = AccountId::from(row.get::<_, u32>(0)?); From 1e12e9d0e6ccb4583a173e4e8e295b9cab38663a Mon Sep 17 00:00:00 2001 From: str4d Date: Wed, 6 Sep 2023 00:18:11 +0100 Subject: [PATCH 0143/1122] Clarify zero-conf shielded note behaviour for `get_wallet_summary` Co-authored-by: Daira Emma Hopwood --- zcash_client_sqlite/src/wallet.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index cc52f40ab9..a4bb4846bc 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -584,6 +584,9 @@ impl ScanProgress for SubtreeScanProgress { /// /// This may be used to obtain a balance that ignores notes that have been detected so recently /// that they are not yet spendable, or for which it is not yet possible to construct witnesses. +/// +/// `min_confirmations` can be 0, but that case is currently treated identically to +/// `min_confirmations == 1` for shielded notes. This behaviour may change in the future. pub(crate) fn get_wallet_summary( conn: &rusqlite::Connection, min_confirmations: u32, From 65420f59df6e554f016ee4c100274ee009c4c993 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 6 Sep 2023 08:47:51 -0600 Subject: [PATCH 0144/1122] zcash_client_backend: Add `WalletRead::block_max_scanned` Fixes #941 --- zcash_client_backend/CHANGELOG.md | 6 +++-- zcash_client_backend/src/data_api.rs | 11 ++++++++ zcash_client_sqlite/src/lib.rs | 4 +++ zcash_client_sqlite/src/wallet.rs | 31 ++++++++++++++++++++--- zcash_client_sqlite/src/wallet/sapling.rs | 9 ++++++- 5 files changed, 55 insertions(+), 6 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 2859d28240..e81dbab777 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -34,8 +34,10 @@ and this library adheres to Rust's notion of - `ShieldedProtocol` - `WalletCommitmentTrees` - `WalletSummary` - - `WalletRead::{chain_height, block_metadata, block_fully_scanned, suggest_scan_ranges, - get_wallet_birthday, get_account_birthday, get_wallet_summary}` + - `WalletRead::{ + chain_height, block_metadata, block_max_scanned, block_fully_scanned, + suggest_scan_ranges, get_wallet_birthday, get_account_birthday, get_wallet_summary + }` - `WalletWrite::{put_blocks, update_chain_tip}` - `chain::CommitmentTreeRoot` - `scanning` A new module containing types required for `suggest_scan_ranges` diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index d42b0da95f..d65cf718a3 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -233,6 +233,13 @@ pub trait WalletRead { /// block. fn block_fully_scanned(&self) -> Result, Self::Error>; + /// Returns block metadata for the maximum height that the wallet has scanned. + /// + /// If the wallet is fully synced, this will be equivalent to `block_fully_scanned`; + /// otherwise the maximal scanned height is likely to be greater than the fully scanned height + /// due to the fact that out-of-order scanning can leave gaps. + fn block_max_scanned(&self) -> Result, Self::Error>; + /// Returns a vector of suggested scan ranges based upon the current wallet state. /// /// This method should only be used in cases where the [`CompactBlock`] data that will be made @@ -945,6 +952,10 @@ pub mod testing { Ok(None) } + fn block_max_scanned(&self) -> Result, Self::Error> { + Ok(None) + } + fn suggest_scan_ranges(&self) -> Result, Self::Error> { Ok(vec![]) } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 670f5f97bd..22f0ff8c68 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -180,6 +180,10 @@ impl, P: consensus::Parameters> WalletRead for W wallet::block_fully_scanned(self.conn.borrow()) } + fn block_max_scanned(&self) -> Result, Self::Error> { + wallet::block_max_scanned(self.conn.borrow()) + } + fn suggest_scan_ranges(&self) -> Result, Self::Error> { wallet::scanning::suggest_scan_ranges(self.conn.borrow(), ScanPriority::Historic) .map_err(SqliteClientError::from) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 5f4faedbb2..97460ede85 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1086,6 +1086,33 @@ pub(crate) fn block_fully_scanned( } } +pub(crate) fn block_max_scanned( + conn: &rusqlite::Connection, +) -> Result, SqliteClientError> { + conn.query_row( + "SELECT blocks.height, hash, sapling_commitment_tree_size, sapling_tree + FROM blocks + JOIN (SELECT MAX(height) AS height FROM blocks) blocks_max + ON blocks.height = blocks_max.height", + [], + |row| { + let height: u32 = row.get(0)?; + let block_hash: Vec = row.get(1)?; + let sapling_tree_size: Option = row.get(2)?; + let sapling_tree: Vec = row.get(3)?; + Ok(( + BlockHeight::from(height), + block_hash, + sapling_tree_size, + sapling_tree, + )) + }, + ) + .optional() + .map_err(SqliteClientError::from) + .and_then(|meta_row| meta_row.map(parse_block_metadata).transpose()) +} + /// Returns the block height at which the specified transaction was mined, /// if any. pub(crate) fn get_tx_height( @@ -1892,8 +1919,6 @@ pub(crate) fn prune_nullifier_map( mod tests { use std::num::NonZeroU32; - use zcash_primitives::transaction::components::Amount; - use zcash_client_backend::data_api::{AccountBirthday, WalletRead}; use crate::{testing::TestBuilder, AccountId}; @@ -1906,7 +1931,7 @@ mod tests { }, zcash_primitives::{ consensus::BlockHeight, - transaction::components::{OutPoint, TxOut}, + transaction::components::{Amount, OutPoint, TxOut}, }, }; diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index d2a5271c11..867c020123 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -465,7 +465,7 @@ pub(crate) mod tests { use crate::{ error::SqliteClientError, testing::{AddressType, BlockCache, TestBuilder, TestState}, - wallet::commitment_tree, + wallet::{block_max_scanned, commitment_tree}, AccountId, NoteId, ReceivedNoteId, }; @@ -501,6 +501,13 @@ pub(crate) mod tests { // Verified balance matches total balance assert_eq!(st.get_total_balance(account), value); + assert_eq!( + block_max_scanned(&st.wallet().conn) + .unwrap() + .unwrap() + .block_height(), + h + ); let to_extsk = ExtendedSpendingKey::master(&[]); let to: RecipientAddress = to_extsk.default_address().1.into(); From c9d58b223c2cb29fbdefc35bc44c5793bfb21425 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 6 Sep 2023 17:13:12 +0000 Subject: [PATCH 0145/1122] zcash_client_sqlite: Fix bug in `check_shard_discontinuity` The previous code was checking that the existing shard range was at least partially inside the proposed insertion range, but this didn't handle the case where the proposed insertion range was fully contained by the existing shard range. --- .../src/wallet/commitment_tree.rs | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 6618cae753..f90b135bea 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -40,8 +40,7 @@ pub enum Error { /// are discontinuous with the existing roots in the database. SubtreeDiscontinuity { attempted_insertion_range: Range, - min: u64, - max: u64, + existing_range: Range, }, } @@ -64,13 +63,12 @@ impl fmt::Display for Error { } Error::SubtreeDiscontinuity { attempted_insertion_range, - min, - max, + existing_range, } => { write!( f, "Attempted to write subtree roots with indicies {:?} which is discontinuous with existing subtree range {:?}", - attempted_insertion_range, *min..(*max+1) + attempted_insertion_range, existing_range, ) } } @@ -395,7 +393,7 @@ fn check_shard_discontinuity( table_prefix: &'static str, proposed_insertion_range: Range, ) -> Result<(), Error> { - if let Ok((Some(min), Some(max))) = conn + if let Ok((Some(stored_min), Some(stored_max))) = conn .query_row( &format!( "SELECT MIN(shard_index), MAX(shard_index) FROM {}_tree_shards", @@ -410,17 +408,20 @@ fn check_shard_discontinuity( ) .map_err(Error::Query) { - if !proposed_insertion_range.contains(&min) && !proposed_insertion_range.contains(&max) { - // The proposed insertion range does not overlap with the existing shard indicies. - // This means a discontinuity is introduced if the proposed insertion range's start - // is not `max + 1` _and_ `min` is not the proposed insertion range's end. - if max + 1 != proposed_insertion_range.start && min != proposed_insertion_range.end { - return Err(Error::SubtreeDiscontinuity { - attempted_insertion_range: proposed_insertion_range, - min, - max, - }); - } + // If the ranges overlap, or are directly adjacent, then we aren't creating a + // discontinuity. We can check this by comparing their start-inclusive, + // end-exclusive bounds: + // - If `cur_start == ins_end` then the proposed insertion range is immediately + // before the current shards. If `cur_start > ins_end` then there is a gap. + // - If `ins_start == cur_end` then the proposed insertion range is immediately + // after the current shards. If `ins_start > cur_end` then there is a gap. + let (cur_start, cur_end) = (stored_min, stored_max + 1); + let (ins_start, ins_end) = (proposed_insertion_range.start, proposed_insertion_range.end); + if cur_start > ins_end || ins_start > cur_end { + return Err(Error::SubtreeDiscontinuity { + attempted_insertion_range: proposed_insertion_range, + existing_range: cur_start..cur_end, + }); } } From f56c5d8f4b191cc365eb753d87d37574529b8ce7 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 6 Sep 2023 11:51:34 -0600 Subject: [PATCH 0146/1122] Measure scan progress from wallet birthday, not the fully scanned height. --- zcash_client_sqlite/src/wallet.rs | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 97460ede85..96400c5bef 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -524,21 +524,22 @@ impl ScanProgress for SubtreeScanProgress { ) .map_err(SqliteClientError::from) } else { - // Compute the number of fully scanned notes directly from the blocks table - let fully_scanned_size = conn.query_row( + let start_height = birthday_height; + // Compute the starting number of notes directly from the blocks table + let start_size = conn.query_row( "SELECT MAX(sapling_commitment_tree_size) FROM blocks - WHERE height <= :fully_scanned_height", - named_params![":fully_scanned_height": u32::from(fully_scanned_height)], + WHERE height <= :start_height", + named_params![":start_height": u32::from(start_height)], |row| row.get::<_, Option>(0), )?; - // Compute the total blocks scanned so far above the fully scanned height + // Compute the total blocks scanned so far above the starting height let scanned_count = conn.query_row( "SELECT SUM(sapling_output_count) FROM blocks - WHERE height > :fully_scanned_height", - named_params![":fully_scanned_height": u32::from(fully_scanned_height)], + WHERE height > :start_height", + named_params![":start_height": u32::from(start_height)], |row| row.get::<_, Option>(0), )?; @@ -552,23 +553,24 @@ impl ScanProgress for SubtreeScanProgress { .query_row( "SELECT MIN(shard_index), MAX(shard_index) FROM sapling_tree_shards - WHERE subtree_end_height > :fully_scanned_height + WHERE subtree_end_height > :start_height OR subtree_end_height IS NULL", - named_params![":fully_scanned_height": u32::from(fully_scanned_height)], + named_params![":start_height": u32::from(start_height)], |row| { let min_tree_size = row .get::<_, Option>(0)? .map(|min| min << SAPLING_SHARD_HEIGHT); let max_idx = row.get::<_, Option>(1)?; - Ok(fully_scanned_size.or(min_tree_size).zip(max_idx).map( - |(min_tree_size, max)| { + Ok(start_size + .or(min_tree_size) + .zip(max_idx) + .map(|(min_tree_size, max)| { let max_tree_size = (max + 1) << SAPLING_SHARD_HEIGHT; Ratio::new( scanned_count.unwrap_or(0), max_tree_size - min_tree_size, ) - }, - )) + })) }, ) .optional()? From a6328a098fb62ce9eddda6375c9aabd376956d57 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 7 Sep 2023 09:58:21 -0600 Subject: [PATCH 0147/1122] zcash_client_sqlite: Bound creation of scan ranges on wallet birthday. This modifies `update_chain_tip` and `scan_complete` to ensure that newly created scan ranged do not extend below the wallet birthday height. Fixes #947 --- zcash_client_sqlite/src/wallet/scanning.rs | 103 +++++++++++---------- 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 7f8a0e82c7..2a0e425cec 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -633,11 +633,15 @@ pub(crate) fn scan_complete( range: Range, wallet_note_positions: &[Position], ) -> Result<(), SqliteClientError> { + // Read the wallet birthday (if known). + let wallet_birthday = wallet_birthday(conn)?; + // Determine the range of block heights for which we will be updating the scan queue. let extended_range = { - // If notes have been detected in the scan, we need to extend any adjacent un-scanned ranges to - // include the blocks needed to complete the note commitment tree subtrees containing the - // positions of the discovered notes. We will query by subtree index to find these bounds. + // If notes have been detected in the scan, we need to extend any adjacent un-scanned ranges + // above the wallet birthday to include the blocks needed to complete the note commitment + // tree subtrees containing the positions of the discovered notes. We will query by subtree + // index to find these bounds. let required_subtrees = wallet_note_positions .iter() .map(|p| Address::above_position(SAPLING_SHARD_HEIGHT.into(), *p).index()) @@ -667,7 +671,8 @@ pub(crate) fn scan_complete( // If no notes belonging to the wallet were found, we don't need to extend the scanning // range suggestions to include the associated subtrees, and our bounds are just the - // scanned range. + // scanned range. Otherwise, ensure that all shard ranges above the wallet birthday + // are included. subtree_bounds .map(|(min_idx, max_idx)| { let range_min = if *min_idx > 0 { @@ -678,6 +683,10 @@ pub(crate) fn scan_complete( params.activation_height(NetworkUpgrade::Sapling) }; + // bound the minimum to the wallet birthday + let range_min = + range_min.map(|h| wallet_birthday.map_or(h, |b| std::cmp::max(b, h))); + // get the block height for the end of the current shard let range_max = sapling_shard_end(*max_idx)?; @@ -758,9 +767,10 @@ pub(crate) fn update_chain_tip( )?; // Create a scanning range for the fragment of the last shard leading up to new tip. - let tip_shard_entry = shard_start_height - .filter(|h| h < &chain_end) - .map(|h| ScanRange::from_parts(h..chain_end, ScanPriority::ChainTip)); + let tip_shard_entry = shard_start_height.filter(|h| h < &chain_end).map(|h| { + let min_to_scan = wallet_birthday.filter(|b| b > &h).unwrap_or(h); + ScanRange::from_parts(min_to_scan..chain_end, ScanPriority::ChainTip) + }); // Create scan ranges to either validate potentially invalid blocks at the wallet's // view of the chain tip, or connect the prior tip to the new tip. @@ -1402,7 +1412,7 @@ mod tests { .with_block_cache() .with_test_account(|network| { // We use Canopy activation as an arbitrary birthday height that's greater than Sapling - // activation. + // activation. We set the Canopy frontier to be 1234 notes into the second shard. let birthday_height = network.activation_height(NetworkUpgrade::Canopy).unwrap(); let frontier_position = Position::from((0x1 << 16) + 1234); let frontier = Frontier::from_parts( @@ -1539,12 +1549,12 @@ mod tests { // Verify that the suggested scan ranges match what is expected. let expected = vec![ - // The entire last (incomplete) shard's range is marked for catching up to the - // chain tip, to ensure that if any notes are discovered after the wallet's + // The last (incomplete) shard's range above the wallet birthday is marked for catching + // up to the chain tip, to ensure that if any notes are discovered after the wallet's // birthday, they will be spendable. - scan_range(last_shard_start.into()..chain_end, ChainTip), - // The range below the last shard is ignored. - scan_range(sap_active..last_shard_start.into(), Ignored), + scan_range(birthday.height().into()..chain_end, ChainTip), + // The range below the birthday height is ignored. + scan_range(sap_active..birthday.height().into(), Ignored), ]; let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); @@ -1555,24 +1565,24 @@ mod tests { fn update_chain_tip_unstable_max_scanned() { use ScanPriority::*; + // this birthday is 1234 notes into the second shard let (mut st, dfvk, birthday, sap_active) = test_with_canopy_birthday(); // Set up the following situation: // - // prior_tip new_tip - // |<--- 500 --->|<- 40 ->|<-- 70 -->|<- 20 ->| - // wallet_birthday max_scanned last_shard_start + // prior_tip new_tip + // |<------ 1000 ------>|<--- 500 --->|<- 40 ->|<-- 70 -->|<- 20 ->| + // initial_shard_end wallet_birthday max_scanned last_shard_start // let max_scanned = birthday.height() + 500; - let prior_tip = max_scanned + 40; // Set up some shard root history before the wallet birthday. - let second_to_last_shard_start = birthday.height() - 1000; + let initial_shard_end = birthday.height() - 1000; st.wallet_mut() .put_sapling_subtree_roots( 0, &[CommitmentTreeRoot::from_parts( - second_to_last_shard_start, + initial_shard_end, // fake a hash, the value doesn't matter Node::empty_leaf(), )], @@ -1581,38 +1591,43 @@ mod tests { // Set up prior chain state. This simulates us having imported a wallet // with a birthday 520 blocks below the chain tip. + let prior_tip = max_scanned + 40; st.wallet_mut().update_chain_tip(prior_tip).unwrap(); // Verify that the suggested scan ranges match what is expected. let expected = vec![ - // The second-to-last shard is currently the last shard, so it is marked for - // scanning to catch up to the prior chain tip. This includes heights prior to - // the wallet's birthday, because the wallet doesn't know that it already has - // enough data from the initial frontier to avoid having to scan this range. - scan_range( - second_to_last_shard_start.into()..(prior_tip + 1).into(), - ChainTip, - ), - // The range below the second-to-last shard is ignored. - scan_range(sap_active..second_to_last_shard_start.into(), Ignored), + scan_range(birthday.height().into()..(prior_tip + 1).into(), ChainTip), + scan_range(sap_active..birthday.height().into(), Ignored), ]; - let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); assert_eq!(actual, expected); - // Now, scan the max scanned block. + // Now, scan the max scanned block. This will trigger the creation of a "FoundNote" range + // in the first shard that extends from the wallet birthday up to st.generate_block_at( max_scanned, BlockHash([0u8; 32]), &dfvk, AddressType::DefaultExternal, Amount::const_from_i64(10000), + // 1235 notes into into the second shard u64::from(birthday.sapling_frontier().value().unwrap().position() + 1) .try_into() .unwrap(), ); st.scan_cached_blocks(max_scanned, 1); + // Verify that the suggested scan ranges match what is expected. + let expected = vec![ + scan_range((max_scanned + 1).into()..(prior_tip + 1).into(), ChainTip), + scan_range(birthday.height().into()..max_scanned.into(), ChainTip), + scan_range(max_scanned.into()..(max_scanned + 1).into(), Scanned), + scan_range(sap_active..birthday.height().into(), Ignored), + ]; + + let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); + assert_eq!(actual, expected); + // Now simulate shutting down, and then restarting 90 blocks later, after a shard // has been completed. let last_shard_start = prior_tip + 70; @@ -1646,16 +1661,13 @@ mod tests { ChainTip, ), // The remainder of the second-to-last shard's range is still in the queue. - scan_range( - second_to_last_shard_start.into()..max_scanned.into(), - ChainTip, - ), + scan_range(birthday.height().into()..max_scanned.into(), ChainTip), // The gap between the prior tip and the last shard is deferred as low priority. scan_range((prior_tip + 1).into()..last_shard_start.into(), Historic), // The max scanned block itself is left as-is. scan_range(max_scanned.into()..(max_scanned + 1).into(), Scanned), // The range below the second-to-last shard is ignored. - scan_range(sap_active..second_to_last_shard_start.into(), Ignored), + scan_range(sap_active..birthday.height().into(), Ignored), ]; let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); @@ -1700,16 +1712,8 @@ mod tests { // Verify that the suggested scan ranges match what is expected. let expected = vec![ - // The second-to-last shard is currently the last shard, so it is marked for - // scanning to catch up to the prior chain tip. This includes heights prior to - // the wallet's birthday, because the wallet doesn't know that it already has - // enough data from the initial frontier to avoid having to scan this range. - scan_range( - second_to_last_shard_start.into()..(prior_tip + 1).into(), - ChainTip, - ), - // The range below the second-to-last shard is ignored. - scan_range(sap_active..second_to_last_shard_start.into(), Ignored), + scan_range(birthday.height().into()..(prior_tip + 1).into(), ChainTip), + scan_range(sap_active..birthday.height().into(), Ignored), ]; let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); @@ -1759,14 +1763,11 @@ mod tests { // The blocks after the max scanned block up to the chain tip are prioritised. scan_range((max_scanned + 1).into()..chain_end, ChainTip), // The remainder of the second-to-last shard's range is still in the queue. - scan_range( - second_to_last_shard_start.into()..max_scanned.into(), - ChainTip, - ), + scan_range(birthday.height().into()..max_scanned.into(), ChainTip), // The max scanned block itself is left as-is. scan_range(max_scanned.into()..(max_scanned + 1).into(), Scanned), // The range below the second-to-last shard is ignored. - scan_range(sap_active..second_to_last_shard_start.into(), Ignored), + scan_range(sap_active..birthday.height().into(), Ignored), ]; let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); From 19582a4ca60a93b6eb454ac1b9a1dea9488f7027 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 7 Sep 2023 10:29:21 -0600 Subject: [PATCH 0148/1122] Apply suggestions from code review Co-authored-by: str4d --- zcash_client_sqlite/src/wallet/scanning.rs | 26 ++++++++++++---------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 2a0e425cec..e291c312ba 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -638,10 +638,10 @@ pub(crate) fn scan_complete( // Determine the range of block heights for which we will be updating the scan queue. let extended_range = { - // If notes have been detected in the scan, we need to extend any adjacent un-scanned ranges - // above the wallet birthday to include the blocks needed to complete the note commitment - // tree subtrees containing the positions of the discovered notes. We will query by subtree - // index to find these bounds. + // If notes have been detected in the scan, we need to extend any adjacent un-scanned + // ranges starting from the wallet birthday to include the blocks needed to complete + // the note commitment tree subtrees containing the positions of the discovered notes. + // We will query by subtree index to find these bounds. let required_subtrees = wallet_note_positions .iter() .map(|p| Address::above_position(SAPLING_SHARD_HEIGHT.into(), *p).index()) @@ -671,8 +671,8 @@ pub(crate) fn scan_complete( // If no notes belonging to the wallet were found, we don't need to extend the scanning // range suggestions to include the associated subtrees, and our bounds are just the - // scanned range. Otherwise, ensure that all shard ranges above the wallet birthday - // are included. + // scanned range. Otherwise, ensure that all shard ranges starting from the wallet + // birthday are included. subtree_bounds .map(|(min_idx, max_idx)| { let range_min = if *min_idx > 0 { @@ -767,6 +767,9 @@ pub(crate) fn update_chain_tip( )?; // Create a scanning range for the fragment of the last shard leading up to new tip. + // We set a lower bound at the wallet birthday (if known), because account creation + // requires specifying a tree frontier that ensures we don't need tree information + // prior to the birthday. let tip_shard_entry = shard_start_height.filter(|h| h < &chain_end).map(|h| { let min_to_scan = wallet_birthday.filter(|b| b > &h).unwrap_or(h); ScanRange::from_parts(min_to_scan..chain_end, ScanPriority::ChainTip) @@ -1549,9 +1552,9 @@ mod tests { // Verify that the suggested scan ranges match what is expected. let expected = vec![ - // The last (incomplete) shard's range above the wallet birthday is marked for catching - // up to the chain tip, to ensure that if any notes are discovered after the wallet's - // birthday, they will be spendable. + // The last (incomplete) shard's range starting from the wallet birthday is + // marked for catching up to the chain tip, to ensure that if any notes are + // discovered after the wallet's birthday, they will be spendable. scan_range(birthday.height().into()..chain_end, ChainTip), // The range below the birthday height is ignored. scan_range(sap_active..birthday.height().into(), Ignored), @@ -1572,7 +1575,7 @@ mod tests { // // prior_tip new_tip // |<------ 1000 ------>|<--- 500 --->|<- 40 ->|<-- 70 -->|<- 20 ->| - // initial_shard_end wallet_birthday max_scanned last_shard_start + // initial_shard_end wallet_birthday max_scanned last_shard_start // let max_scanned = birthday.height() + 500; @@ -1602,8 +1605,7 @@ mod tests { let actual = suggest_scan_ranges(&st.wallet().conn, Ignored).unwrap(); assert_eq!(actual, expected); - // Now, scan the max scanned block. This will trigger the creation of a "FoundNote" range - // in the first shard that extends from the wallet birthday up to + // Now, scan the max scanned block. st.generate_block_at( max_scanned, BlockHash([0u8; 32]), From 0937c6a38407a9c98f546c43dc96ffdc6dece48e Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 6 Sep 2023 16:12:54 -0600 Subject: [PATCH 0149/1122] zcash_client_sqlite: Add a test demonstrating chain tip fund unavailability. This also factors out and cleans up repeated code that checks for anchor shard completeness. --- zcash_client_sqlite/src/wallet/sapling.rs | 147 +++++++++++++++++---- zcash_client_sqlite/src/wallet/scanning.rs | 4 +- 2 files changed, 126 insertions(+), 25 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 867c020123..d29c92b813 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -126,6 +126,30 @@ fn to_spendable_note(row: &Row) -> Result, S }) } +/// Utility method for determining whether we have any spendable notes +/// +/// If the tip shard has unscanned ranges below the anchor height and greater than or equal to +/// the wallet birthday, none of our notes can be spent because we cannot construct witnesses at +/// the provided anchor height. +fn unscanned_tip_exists( + conn: &Connection, + anchor_height: BlockHeight, +) -> Result { + // v_sapling_shard_unscanned_ranges only returns ranges ending on or after wallet birthday, so + // we don't need to refer to the birthday in this query. + conn.query_row( + "SELECT EXISTS ( + SELECT 1 FROM v_sapling_shard_unscanned_ranges range + WHERE range.block_range_start <= :anchor_height + AND :anchor_height BETWEEN + range.subtree_start_height + AND IFNULL(range.subtree_end_height, :anchor_height) + )", + named_params![":anchor_height": u32::from(anchor_height),], + |row| row.get::<_, bool>(0), + ) +} + pub(crate) fn get_spendable_sapling_notes( conn: &Connection, account: AccountId, @@ -141,16 +165,7 @@ pub(crate) fn get_spendable_sapling_notes( } }; - let mut stmt_unscanned_tip = conn.prepare_cached( - "SELECT 1 FROM v_sapling_shard_unscanned_ranges - WHERE :anchor_height BETWEEN subtree_start_height AND IFNULL(subtree_end_height, :anchor_height) - AND block_range_start <= :anchor_height", - )?; - let mut unscanned = - stmt_unscanned_tip.query(named_params![":anchor_height": &u32::from(anchor_height),])?; - if unscanned.next()?.is_some() { - // if the tip shard has unscanned ranges below the anchor height, none of our notes can be - // spent + if unscanned_tip_exists(conn, anchor_height)? { return Ok(vec![]); } @@ -207,16 +222,7 @@ pub(crate) fn select_spendable_sapling_notes( } }; - let mut stmt_unscanned_tip = conn.prepare_cached( - "SELECT 1 FROM v_sapling_shard_unscanned_ranges - WHERE :anchor_height BETWEEN subtree_start_height AND IFNULL(subtree_end_height, :anchor_height) - AND block_range_start <= :anchor_height", - )?; - let mut unscanned = - stmt_unscanned_tip.query(named_params![":anchor_height": &u32::from(anchor_height),])?; - if unscanned.next()?.is_some() { - // if the tip shard has unscanned ranges below the anchor height, none of our notes can be - // spent + if unscanned_tip_exists(conn, anchor_height)? { return Ok(vec![]); } @@ -427,14 +433,17 @@ pub(crate) fn put_received_note( pub(crate) mod tests { use std::{convert::Infallible, num::NonZeroU32}; + use incrementalmerkletree::Hashable; use zcash_proofs::prover::LocalTxProver; use zcash_primitives::{ + block::BlockHash, consensus::BranchId, legacy::TransparentAddress, memo::{Memo, MemoBytes}, sapling::{ - note_encryption::try_sapling_output_recovery, prover::TxProver, Note, PaymentAddress, + note_encryption::try_sapling_output_recovery, prover::TxProver, Node, Note, + PaymentAddress, }, transaction::{ components::{ @@ -451,9 +460,10 @@ pub(crate) mod tests { address::RecipientAddress, data_api::{ self, + chain::CommitmentTreeRoot, error::Error, wallet::input_selection::{GreedyInputSelector, GreedyInputSelectorError}, - AccountBirthday, Ratio, ShieldedProtocol, WalletRead, + AccountBirthday, Ratio, ShieldedProtocol, WalletCommitmentTrees, WalletRead, }, decrypt_transaction, fees::{fixed, zip317, DustOutputPolicy}, @@ -465,7 +475,10 @@ pub(crate) mod tests { use crate::{ error::SqliteClientError, testing::{AddressType, BlockCache, TestBuilder, TestState}, - wallet::{block_max_scanned, commitment_tree}, + wallet::{ + block_max_scanned, commitment_tree, sapling::select_spendable_sapling_notes, + scanning::tests::test_with_canopy_birthday, + }, AccountId, NoteId, ReceivedNoteId, }; @@ -1203,4 +1216,92 @@ pub(crate) mod tests { Ok(_) ); } + + #[test] + fn birthday_in_anchor_shard() { + let (mut st, dfvk, birthday, _) = test_with_canopy_birthday(); + + // Set up the following situation: + // + // |<------ 500 ------->|<--- 10 --->|<--- 10 --->| + // last_shard_start wallet_birthday received_tx anchor_height + // + // Set up some shard root history before the wallet birthday. + let prev_shard_start = birthday.height() - 500; + st.wallet_mut() + .put_sapling_subtree_roots( + 0, + &[CommitmentTreeRoot::from_parts( + prev_shard_start, + // fake a hash, the value doesn't matter + Node::empty_leaf(), + )], + ) + .unwrap(); + + let received_tx_height = birthday.height() + 10; + + let initial_sapling_tree_size = + u64::from(birthday.sapling_frontier().value().unwrap().position() + 1) + .try_into() + .unwrap(); + + // Generate 9 blocks that have no value for us, starting at the birthday height. + let not_our_key = ExtendedSpendingKey::master(&[]).to_diversifiable_full_viewing_key(); + let not_our_value = Amount::const_from_i64(10000); + st.generate_block_at( + birthday.height(), + BlockHash([0; 32]), + ¬_our_key, + AddressType::DefaultExternal, + not_our_value, + initial_sapling_tree_size, + ); + for _ in 1..9 { + st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); + } + + // Now, generate a block that belongs to our wallet + st.generate_next_block( + &dfvk, + AddressType::DefaultExternal, + Amount::const_from_i64(500000), + ); + + // Generate some more blocks to get above our anchor height + for _ in 0..15 { + st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); + } + + // Scan a block range that includes our received note, but skips some blocks we need to + // make it spendable. + st.scan_cached_blocks(birthday.height() + 5, 20); + + // Verify that the received note is not considered spendable + let spendable = select_spendable_sapling_notes( + &st.wallet().conn, + AccountId::from(0), + Amount::const_from_i64(300000), + received_tx_height + 10, + &[], + ) + .unwrap(); + + assert_eq!(spendable.len(), 0); + + // Scan the blocks we skipped + st.scan_cached_blocks(birthday.height(), 5); + + // Verify that the received note is now considered spendable + let spendable = select_spendable_sapling_notes( + &st.wallet().conn, + AccountId::from(0), + Amount::const_from_i64(300000), + received_tx_height + 10, + &[], + ) + .unwrap(); + + assert_eq!(spendable.len(), 1); + } } diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 7f8a0e82c7..9b11509785 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -872,7 +872,7 @@ pub(crate) fn update_chain_tip( } #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::ops::Range; use incrementalmerkletree::{frontier::Frontier, Hashable, Level, Position}; @@ -1392,7 +1392,7 @@ mod tests { ); } - fn test_with_canopy_birthday() -> ( + pub(crate) fn test_with_canopy_birthday() -> ( TestState, DiversifiableFullViewingKey, AccountBirthday, From 2131cd547e56b9de836d431c72890d41e7851837 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 7 Sep 2023 10:39:00 -0600 Subject: [PATCH 0150/1122] zcash_client_sqlite: Return a balance for every account from `get_wallet_summary` Fixes #948 --- zcash_client_backend/src/data_api.rs | 6 +++ zcash_client_sqlite/src/wallet.rs | 57 ++++++++++++---------------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index d65cf718a3..7cb5d6013d 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -102,6 +102,12 @@ pub struct AccountBalance { } impl AccountBalance { + /// The [`Balance`] value having zero values for all its fields. + pub const ZERO: Self = Self { + sapling_balance: Balance::ZERO, + unshielded: NonNegativeAmount::ZERO, + }; + /// Returns the total value of funds belonging to the account. pub fn total(&self) -> NonNegativeAmount { (self.sapling_balance.total() + self.unshielded) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 96400c5bef..bcc1d4f9f2 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -73,7 +73,7 @@ use std::convert::TryFrom; use std::io::{self, Cursor}; use std::num::NonZeroU32; use tracing::debug; -use zcash_client_backend::data_api::{AccountBalance, Balance, Ratio, WalletSummary}; +use zcash_client_backend::data_api::{AccountBalance, Ratio, WalletSummary}; use zcash_primitives::transaction::components::amount::NonNegativeAmount; use zcash_client_backend::data_api::{ @@ -626,6 +626,15 @@ pub(crate) fn get_wallet_summary( |row| row.get::<_, bool>(0), )?; + let mut stmt_accounts = conn.prepare_cached("SELECT account FROM accounts")?; + let mut account_balances = stmt_accounts + .query([])? + .mapped(|row| { + row.get::<_, u32>(0) + .map(|a| (AccountId::from(a), AccountBalance::ZERO)) + }) + .collect::, _>>()?; + let mut stmt_select_notes = conn.prepare_cached( "SELECT n.account, n.value, n.is_change, scan_state.max_priority, t.block FROM sapling_received_notes n @@ -641,7 +650,6 @@ pub(crate) fn get_wallet_summary( )", )?; - let mut account_balances: BTreeMap = BTreeMap::new(); let mut rows = stmt_select_notes.query(named_params![":summary_height": u32::from(summary_height)])?; while let Some(row) = rows.next()? { @@ -689,27 +697,17 @@ pub(crate) fn get_wallet_summary( } }; - account_balances - .entry(account) - .and_modify(|bal| { - bal.sapling_balance.spendable_value = (bal.sapling_balance.spendable_value - + spendable_value) - .expect("Spendable value cannot overflow"); - bal.sapling_balance.change_pending_confirmation = - (bal.sapling_balance.change_pending_confirmation + change_pending_confirmation) - .expect("Pending change value cannot overflow"); - bal.sapling_balance.value_pending_spendability = - (bal.sapling_balance.value_pending_spendability + value_pending_spendability) - .expect("Value pending spendability cannot overflow"); - }) - .or_insert(AccountBalance { - sapling_balance: Balance { - spendable_value, - change_pending_confirmation, - value_pending_spendability, - }, - unshielded: NonNegativeAmount::ZERO, - }); + account_balances.entry(account).and_modify(|bal| { + bal.sapling_balance.spendable_value = (bal.sapling_balance.spendable_value + + spendable_value) + .expect("Spendable value cannot overflow"); + bal.sapling_balance.change_pending_confirmation = + (bal.sapling_balance.change_pending_confirmation + change_pending_confirmation) + .expect("Pending change value cannot overflow"); + bal.sapling_balance.value_pending_spendability = + (bal.sapling_balance.value_pending_spendability + value_pending_spendability) + .expect("Value pending spendability cannot overflow"); + }); } #[cfg(feature = "transparent-inputs")] @@ -734,16 +732,9 @@ pub(crate) fn get_wallet_summary( SqliteClientError::CorruptedData(format!("Negative UTXO value {:?}", raw_value)) })?; - account_balances - .entry(account) - .and_modify(|bal| { - bal.unshielded = - (bal.unshielded + value).expect("Unshielded value cannot overflow") - }) - .or_insert(AccountBalance { - sapling_balance: Balance::ZERO, - unshielded: value, - }); + account_balances.entry(account).and_modify(|bal| { + bal.unshielded = (bal.unshielded + value).expect("Unshielded value cannot overflow") + }); } } From b54969953fc39547e1387e3dff6b965a342d0fbc Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 8 Sep 2023 03:43:07 +0000 Subject: [PATCH 0151/1122] zcash_client_sqlite: Simplify range query in `replace_queue_entries` Co-authored-by: Kris Nuttycombe --- zcash_client_sqlite/src/wallet/scanning.rs | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index b96c842df5..17868e00c2 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -546,21 +546,8 @@ pub(crate) fn replace_queue_entries( .prepare_cached( "SELECT block_range_start, block_range_end, priority FROM scan_queue - WHERE ( - -- the start is contained within or adjacent to the range - :start >= block_range_start - AND :start <= block_range_end - ) - OR ( - -- the end is contained within or adjacent to the range - :end >= block_range_start - AND :end <= block_range_end - ) - OR ( - -- start..end contains the entire range - block_range_start >= :start - AND block_range_end <= :end - ) + -- Ignore ranges that do not overlap and are not adjacent to the query range. + WHERE NOT (block_range_start > :end OR :start > block_range_end) ORDER BY block_range_end", ) .map_err(E::db_error)?; From 7a4954c242b8786873ab9304fc2e7b41f1dc7e39 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 8 Sep 2023 03:44:03 +0000 Subject: [PATCH 0152/1122] zcash_client_sqlite: Fix off-by-one in `scan_complete` Shard end heights are end-inclusive, while Rust `Range`s are end-exclusive. Co-authored-by: Kris Nuttycombe --- zcash_client_sqlite/src/wallet/scanning.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 17868e00c2..8aad71a404 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -674,8 +674,9 @@ pub(crate) fn scan_complete( let range_min = range_min.map(|h| wallet_birthday.map_or(h, |b| std::cmp::max(b, h))); - // get the block height for the end of the current shard - let range_max = sapling_shard_end(*max_idx)?; + // Get the block height for the end of the current shard, and make it an + // exclusive end bound. + let range_max = sapling_shard_end(*max_idx)?.map(|end| end + 1); Ok::, rusqlite::Error>(Range { start: range.start.min(range_min.unwrap_or(range.start)), From 457c9d26ddf92d74db06f5574f615e8364a8206a Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 8 Sep 2023 03:45:05 +0000 Subject: [PATCH 0153/1122] zcash_client_sqlite: Prevent empty `FoundNote` ranges in `scan_complete` In the sandblasting block ranges, shard trees only cover a few hundred blocks at most. When scanning block ranges much larger than this, it is likely that when a note is discovered, its parent shard tree is entirely contained within the scan range. In this situation, `extended_range` would be set to `range`, and then because an extended range existed, ranges with `FoundNote` priority would always be created - that in this case are empty. In an effectively-linear-scanning wallet situation, this leads to a `SpanningTree` being constructed with adjacent `Scanned` ranges, separated by empty `FoundNote` ranges, which it was unable to merge. We address this by both preventing generation of the empty `FoundNote` ranges, and by defensively fixing `SpanningTree::into_vec` to skip empty ranges. Co-authored-by: Kris Nuttycombe --- zcash_client_sqlite/src/wallet/scanning.rs | 68 ++++++++++++++++------ 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 8aad71a404..d1ba239cda 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -436,17 +436,19 @@ impl SpanningTree { fn go(acc: &mut Vec, tree: SpanningTree) { match tree { SpanningTree::Leaf(entry) => { - if let Some(top) = acc.pop() { - match join_nonoverlapping(top, entry) { - Joined::One(merged) => acc.push(merged), - Joined::Two(l, r) => { - acc.push(l); - acc.push(r); + if !entry.is_empty() { + if let Some(top) = acc.pop() { + match join_nonoverlapping(top, entry) { + Joined::One(merged) => acc.push(merged), + Joined::Two(l, r) => { + acc.push(l); + acc.push(r); + } + _ => unreachable!(), } - _ => unreachable!(), + } else { + acc.push(entry); } - } else { - acc.push(entry); } } SpanningTree::Parent { left, right, .. } => { @@ -690,19 +692,26 @@ pub(crate) fn scan_complete( let query_range = extended_range.clone().unwrap_or_else(|| range.clone()); let scanned = ScanRange::from_parts(range.clone(), ScanPriority::Scanned); - let extensions = if let Some(extended) = extended_range { - vec![ - ScanRange::from_parts(extended.start..range.start, ScanPriority::FoundNote), - ScanRange::from_parts(range.end..extended.end, ScanPriority::FoundNote), - ] - } else { - vec![] - }; + + // If any of the extended range actually extends beyond the scanned range, we need to + // scan that extension in order to make the found note(s) spendable. We need to avoid + // creating empty ranges here, as that acts as an optimization barrier preventing + // `SpanningTree` from merging non-empty scanned ranges on either side. + let extended_before = extended_range + .as_ref() + .map(|extended| ScanRange::from_parts(extended.start..range.start, ScanPriority::FoundNote)) + .filter(|range| !range.is_empty()); + let extended_after = extended_range + .map(|extended| ScanRange::from_parts(range.end..extended.end, ScanPriority::FoundNote)) + .filter(|range| !range.is_empty()); replace_queue_entries::( conn, &query_range, - Some(scanned).into_iter().chain(extensions.into_iter()), + Some(scanned) + .into_iter() + .chain(extended_before) + .chain(extended_after), false, )?; @@ -1095,6 +1104,29 @@ pub(crate) mod tests { ); } + #[test] + fn spanning_tree_insert_empty() { + use ScanPriority::*; + + let t = spanning_tree(&[ + (0..3, Historic), + (3..6, Scanned), + (6..6, FoundNote), + (6..8, Scanned), + (8..10, ChainTip), + ]) + .unwrap(); + + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..3, Historic), + scan_range(3..8, Scanned), + scan_range(8..10, ChainTip), + ] + ); + } + #[test] fn spanning_tree_insert_gaps() { use ScanPriority::*; From 504efcfab77bf65583f2d88daf2447c2df5547b5 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 8 Sep 2023 04:04:10 +0000 Subject: [PATCH 0154/1122] zcash_client_sqlite: Fix scan range matching with incomplete shard The `v_sapling_shard_scan_ranges` view pairs every scan range with every shard range, such that each row shows an overlapping pair. For the complete shards, this is an overlap check between two ranges, which the previous query was performing correctly (if verbosely). For the last incomplete shard, we have a half-open range that needs to be handled separately. The previous query only handled the case where a scan range was contained within the last shard, and did not handle the case where the scan range contained the last shard. This led to a puzzling bug, where `WalletDb::get_wallet_summary` was sometimes treating any note received within the last shard as part of the wallet's pending balance. If the wallet's scan queue contained a range that encompassed the last incomplete shard, the bug in the `v_sapling_shard_scan_ranges` view meant that it omitted any mention of the last shard, which translated into these notes being considered unmined when joining `sapling_received_notes` against the sub-view `v_sapling_shards_scan_state`. The bug was made harder to diagnose due to the previous commit's bug that was causing scan ranges to not be correctly merged; this resulted in smaller scan ranges that were more likely to be contained within the last shard, making it visible in `v_sapling_shard_scan_ranges` and enabling notes to be detected as mined. The fixed view uses a simpler query that enables us to handle complete and incomplete shards together. Time spent investigating and fixing: 4.5 hours Co-authored-by: Kris Nuttycombe --- zcash_client_sqlite/src/wallet/init.rs | 14 +++++++------- .../migrations/v_sapling_shard_unscanned_ranges.rs | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index a8ba6a80d2..18c27f303e 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -383,14 +383,14 @@ mod tests { FROM sapling_tree_shards shard LEFT OUTER JOIN sapling_tree_shards prev_shard ON shard.shard_index = prev_shard.shard_index + 1 - INNER JOIN scan_queue ON - (scan_queue.block_range_start >= subtree_start_height AND shard.subtree_end_height IS NULL) OR - (scan_queue.block_range_start BETWEEN subtree_start_height AND shard.subtree_end_height) OR - ((scan_queue.block_range_end - 1) BETWEEN subtree_start_height AND shard.subtree_end_height) OR + -- Join with scan ranges that overlap with the subtree's involved blocks. + INNER JOIN scan_queue ON ( + subtree_start_height < scan_queue.block_range_end AND ( - scan_queue.block_range_start <= prev_shard.subtree_end_height - AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height - )", + scan_queue.block_range_start <= shard.subtree_end_height OR + shard.subtree_end_height IS NULL + ) + )", u32::from(st.network().activation_height(NetworkUpgrade::Sapling).unwrap()), ), // v_sapling_shard_unscanned_ranges diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs index 7a72942c09..dc19265d23 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs @@ -52,14 +52,14 @@ impl RusqliteMigration for Migration

{ FROM sapling_tree_shards shard LEFT OUTER JOIN sapling_tree_shards prev_shard ON shard.shard_index = prev_shard.shard_index + 1 - INNER JOIN scan_queue ON - (scan_queue.block_range_start >= subtree_start_height AND shard.subtree_end_height IS NULL) OR - (scan_queue.block_range_start BETWEEN subtree_start_height AND shard.subtree_end_height) OR - ((scan_queue.block_range_end - 1) BETWEEN subtree_start_height AND shard.subtree_end_height) OR + -- Join with scan ranges that overlap with the subtree's involved blocks. + INNER JOIN scan_queue ON ( + subtree_start_height < scan_queue.block_range_end AND ( - scan_queue.block_range_start <= prev_shard.subtree_end_height - AND (scan_queue.block_range_end - 1) >= shard.subtree_end_height - )", + scan_queue.block_range_start <= shard.subtree_end_height OR + shard.subtree_end_height IS NULL + ) + )", SAPLING_SHARD_HEIGHT, SAPLING_SHARD_HEIGHT, u32::from(self.params.activation_height(NetworkUpgrade::Sapling).unwrap()), From 3a7ef20947a4b5292d2016da422c924f45fc08b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Sep 2023 05:24:37 +0000 Subject: [PATCH 0155/1122] build(deps): bump actions/cache from 3.3.1 to 3.3.2 Bumps [actions/cache](https://github.com/actions/cache) from 3.3.1 to 3.3.2. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3.3.1...v3.3.2) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d03e3d29f..99babd8fc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: run: echo "ZCASH_PARAMS=$(cargo run --release --example get-params-path --features directories)" >> $GITHUB_ENV - name: Cache Zcash parameters id: cache-params - uses: actions/cache@v3.3.1 + uses: actions/cache@v3.3.2 with: path: ${{ env.ZCASH_PARAMS }} key: ${{ runner.os }}-params @@ -109,7 +109,7 @@ jobs: run: echo "ZCASH_PARAMS=$(cargo run --release --example get-params-path --features directories)" >> $GITHUB_ENV - name: Cache Zcash parameters id: cache-params - uses: actions/cache@v3.3.1 + uses: actions/cache@v3.3.2 with: path: ${{ env.ZCASH_PARAMS }} key: ${{ runner.os }}-params From aa039819df19175624b177b1da2ea5ed9ba0c2c7 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 8 Sep 2023 08:46:20 -0600 Subject: [PATCH 0156/1122] Fix formatting & clippy lints. --- zcash_client_sqlite/src/testing.rs | 2 ++ .../v_sapling_shard_unscanned_ranges.rs | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 0034ff4bbb..9074792944 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -434,6 +434,7 @@ impl TestState { /// Invokes [`propose_shielding`] with the given arguments. #[cfg(feature = "transparent-inputs")] #[allow(clippy::type_complexity)] + #[allow(dead_code)] pub(crate) fn propose_shielding( &mut self, input_selector: &InputsT, @@ -576,6 +577,7 @@ impl TestState { .unwrap() } + #[allow(dead_code)] pub(crate) fn get_pending_change( &self, account: AccountId, diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs index dc19265d23..2191944fec 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_sapling_shard_unscanned_ranges.rs @@ -36,9 +36,8 @@ impl RusqliteMigration for Migration

{ type Error = WalletMigrationError; fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { - transaction.execute_batch( - &format!( - "CREATE VIEW v_sapling_shard_scan_ranges AS + transaction.execute_batch(&format!( + "CREATE VIEW v_sapling_shard_scan_ranges AS SELECT shard.shard_index, shard.shard_index << {} AS start_position, @@ -60,11 +59,14 @@ impl RusqliteMigration for Migration

{ shard.subtree_end_height IS NULL ) )", - SAPLING_SHARD_HEIGHT, - SAPLING_SHARD_HEIGHT, - u32::from(self.params.activation_height(NetworkUpgrade::Sapling).unwrap()), - ) - )?; + SAPLING_SHARD_HEIGHT, + SAPLING_SHARD_HEIGHT, + u32::from( + self.params + .activation_height(NetworkUpgrade::Sapling) + .unwrap() + ), + ))?; transaction.execute_batch(&format!( "CREATE VIEW v_sapling_shard_unscanned_ranges AS From 0ea2290670562b8114212e1bd4e7894943eb73f9 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 8 Sep 2023 08:54:39 -0600 Subject: [PATCH 0157/1122] zcash_primitives: update CHANGELOG.md in preparation for the 0.13.0 release --- zcash_primitives/CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index ee6d3861bc..8ab995dea4 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -21,6 +21,9 @@ and this library adheres to Rust's notion of - `TryFrom for NonNegativeAmount` - `Add for NonNegativeAmount` - `Add for Option` + - `Sub for NonNegativeAmount` + - `Sub for Option` +- `zcash_primitives::block::BlockHash::try_from_slice` ### Changed - `zcash_primitives::transaction`: @@ -37,6 +40,14 @@ and this library adheres to Rust's notion of `orchard::builder::InProgress` instead of `zcash_primitives::transaction::components::orchard::Unauthorized` - `zcash_primitives::consensus::NetworkUpgrade` now implements `PartialEq`, `Eq` +- `zcash_primitives::legacy::OpCode` now defines a complete set of transparent opcodes, + instead of just a minimal set. +- `zcash_primitives::legacy::Script` now has a custom `Debug` implementation that + renders script details in a much more legible fashion. +- `zcash_primitives::sapling::redjubjub::Signature` now has a custom `Debug` + implementation that renders details in a much more legible fashion. +- `zcash_primitives::sapling::tree::Node` now has a custom `Debug` + implementation that renders details in a much more legible fashion. ### Removed - `impl {PartialEq, Eq} for transaction::builder::Error` From 1fe763de5f8dfa97bcafcea81aa6cd6cd2864199 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 8 Sep 2023 10:15:49 -0600 Subject: [PATCH 0158/1122] zcash_client_backend: Update CHANGELOG.md for 0.10.0 release. --- zcash_client_backend/CHANGELOG.md | 18 ++++++++++++------ zcash_client_backend/src/data_api.rs | 4 ++++ zcash_client_backend/src/scanning.rs | 2 -- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index e81dbab777..df8a702166 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -10,6 +10,8 @@ and this library adheres to Rust's notion of ### Notable Changes - `zcash_client_backend` now supports out-of-order scanning of blockchain history. + See the module documentation for `zcash_client_backend::data_api::chain` + for details on how to make use of the new scanning capabilities. - This release of `zcash_client_backend` defines the concept of an account birthday. The account birthday is defined as the minimum height among blocks to be scanned when recovering an account. @@ -26,6 +28,7 @@ and this library adheres to Rust's notion of - `AccountBalance` - `AccountBirthday` - `Balance` + - `BirthdayError` - `BlockMetadata` - `NoteId` - `NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` @@ -42,7 +45,8 @@ and this library adheres to Rust's notion of - `chain::CommitmentTreeRoot` - `scanning` A new module containing types required for `suggest_scan_ranges` - `testing::MockWalletDb::new` - - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}`: + - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}` + - `SAPLING_SHARD_HEIGHT` constant - `zcash_client_backend::wallet::WalletSaplingOutput::note_commitment_tree_position` - `zcash_client_backend::scanning`: - `ScanError` @@ -53,8 +57,8 @@ and this library adheres to Rust's notion of ### Changed - MSRV is now 1.65.0. -- Bumped dependencies to `hdwallet 0.4`, `zcash_primitives 0.12`, `zcash_note_encryption 0.4`, - `incrementalmerkletree 0.4`, `orchard 0.5`, `bs58 0.5` +- Bumped dependencies to `hdwallet 0.4`, `zcash_primitives 0.13`, `zcash_note_encryption 0.4`, + `incrementalmerkletree 0.4`, `orchard 0.5`, `bs58 0.5`, `tempfile 3.5.0` - `zcash_client_backend::data_api`: - `WalletRead::TxRef` has been removed in favor of consistently using `TxId` instead. - `WalletRead::get_transaction` now takes a `TxId` as its argument. @@ -84,9 +88,9 @@ and this library adheres to Rust's notion of - `wallet::{spend, create_spend_to_address, shield_transparent_funds, propose_transfer, propose_shielding, create_proposed_transaction}` now take their respective `min_confirmations` arguments as `NonZeroU32` - - `wallet::input_selection::InputSelector::{propose_transaction, propose_shielding}` - now take their respective `min_confirmations` arguments as `NonZeroU32` - - A new `Scan` variant has been added to `data_api::chain::error::Error`. + - A new `Scan` variant replaces the `Chain` variant of `data_api::chain::error::Error`. + The `NoteRef` parameter to `data_api::chain::error::Error` has been removed + in favor of using `NoteId` to report the specific note for which a failure occurred. - A new `SyncRequired` variant has been added to `data_api::wallet::input_selection::InputSelectorError`. - The variants of the `PoolType` enum have changed; the `PoolType::Sapling` variant has been removed in favor of a `PoolType::Shielded` variant that wraps a `ShieldedProtocol` value. @@ -95,6 +99,8 @@ and this library adheres to Rust's notion of - Arguments to `WalletSaplingOutput::from_parts` have changed. - `zcash_client_backend::data_api::wallet::input_selection::InputSelector`: - Arguments to `{propose_transaction, propose_shielding}` have changed. + - `InputSelector::{propose_transaction, propose_shielding}` + now take their respective `min_confirmations` arguments as `NonZeroU32` - `zcash_client_backend::data_api::wallet::{create_spend_to_address, spend, create_proposed_transaction, shield_transparent_funds}` now return the `TxId` for the newly created transaction instead an internal database identifier. diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 7cb5d6013d..a8e83b6ee6 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -42,6 +42,10 @@ pub mod error; pub mod scanning; pub mod wallet; +/// The height of subtree roots in the Sapling note commitment tree. +/// +/// This conforms to the structure of subtree data returned by from +/// `lightwalletd` when using the `GetSubtreeRoots` GRPC call. pub const SAPLING_SHARD_HEIGHT: u8 = sapling::NOTE_COMMITMENT_TREE_DEPTH / 2; /// An enumeration of constraints that can be applied when querying for nullifiers for notes diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index f8ad561548..ba72591844 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -1,6 +1,4 @@ //! Tools for scanning a compact representation of the Zcash block chain. -//! -//! TODO: rename this module to `block_scanner` use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; From 6e34e2d1138bfab94e82feed4d72ba361a8635c9 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 8 Sep 2023 16:21:34 +0000 Subject: [PATCH 0159/1122] zcash_client_backend: Migrate to `prost 0.12`, `tonic 0.10` --- zcash_client_backend/CHANGELOG.md | 3 ++- zcash_client_backend/Cargo.toml | 6 +++--- zcash_client_sqlite/CHANGELOG.md | 2 +- zcash_client_sqlite/Cargo.toml | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index df8a702166..1ec9646004 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -58,7 +58,8 @@ and this library adheres to Rust's notion of ### Changed - MSRV is now 1.65.0. - Bumped dependencies to `hdwallet 0.4`, `zcash_primitives 0.13`, `zcash_note_encryption 0.4`, - `incrementalmerkletree 0.4`, `orchard 0.5`, `bs58 0.5`, `tempfile 3.5.0` + `incrementalmerkletree 0.4`, `orchard 0.5`, `bs58 0.5`, `tempfile 3.5.0`, `prost 0.12`, + `tonic 0.10`. - `zcash_client_backend::data_api`: - `WalletRead::TxRef` has been removed in favor of consistently using `TxId` instead. - `WalletRead::get_transaction` now takes a `TxId` as its argument. diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 8e6d876ffa..3757f3e2d7 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -46,8 +46,8 @@ tracing = "0.1" # - Protobuf interfaces and gRPC bindings hex = "0.4" -prost = "0.11" -tonic = { version = "0.9", optional = true } +prost = "0.12" +tonic = { version = "0.10", optional = true } # - Secret management secrecy = "0.8" @@ -75,7 +75,7 @@ crossbeam-channel = "0.5" rayon = "1.5" [build-dependencies] -tonic-build = "0.9" +tonic-build = "0.10" which = "4" [dev-dependencies] diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 378e0381f9..11f403a7c0 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -19,7 +19,7 @@ and this library adheres to Rust's notion of ### Changed - MSRV is now 1.65.0. - Bumped dependencies to `hdwallet 0.4`, `incrementalmerkletree 0.4`, `bs58 0.5`, - `zcash_primitives 0.12` + `zcash_primitives 0.12`, `prost 0.12`. - A `CommitmentTree` variant has been added to `zcash_client_sqlite::wallet::init::WalletMigrationError` - `min_confirmations` parameter values are now more strongly enforced. Previously, a note could be spent with fewer than `min_confirmations` confirmations if the diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 3494b19968..d1f31874ff 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -31,7 +31,7 @@ tracing = "0.1" # - Serialization byteorder = "1" -prost = "0.11" +prost = "0.12" group = "0.13" jubjub = "0.10" From 61e06d2227aa3c40f05f9caf38983912e2aaacd3 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 8 Sep 2023 11:00:54 -0600 Subject: [PATCH 0160/1122] zcash_client_sqlite: Update CHANGELOG.md for 0.8.0 release --- zcash_client_sqlite/CHANGELOG.md | 15 +++++++++++++-- .../init/migrations/sapling_memo_consistency.rs | 10 +++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 11f403a7c0..4aeedfc27a 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -7,6 +7,8 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added +- `zcash_client_sqlite::commitment_tree` Types related to management of note + commitment trees using the `shardtree` crate. - `zcash_client_sqlite::serialization` Serialization formats for data stored as SQLite BLOBs in the wallet database. - A new default-enabled feature flag `multicore`. This allows users to disable @@ -15,11 +17,15 @@ and this library adheres to Rust's notion of - `zcash_client_sqlite::ReceivedNoteId` - `zcash_client_sqlite::wallet::commitment_tree` A new module containing a sqlite-backed implementation of `shardtree::store::ShardStore`. +- `impl zcash_client_backend::data_api::WalletCommitmentTrees for WalletDb` ### Changed - MSRV is now 1.65.0. -- Bumped dependencies to `hdwallet 0.4`, `incrementalmerkletree 0.4`, `bs58 0.5`, - `zcash_primitives 0.12`, `prost 0.12`. +- Bumped dependencies to `hdwallet 0.4`, `incrementalmerkletree 0.5`, `bs58 0.5`, + `zcash_primitives 0.12`, `rusqlite 0.29`, `schemer-rusqlite 0.2.2`, `time 0.3.22`, + `tempfile 3.5`, `zcash_note_encryption 0.4`, `zcash_proofs 0.12`, `prost 0.12` + `zcash_client_backend 0.9`, `zcash_address 0.3` +- Added dependencies on `shardtree 0.0`, `zcash_encoding 0.2`, `byteorder 1` - A `CommitmentTree` variant has been added to `zcash_client_sqlite::wallet::init::WalletMigrationError` - `min_confirmations` parameter values are now more strongly enforced. Previously, a note could be spent with fewer than `min_confirmations` confirmations if the @@ -30,6 +36,8 @@ and this library adheres to Rust's notion of - `SqliteClientError::BlockConflict` - `SqliteClientError::CacheMiss` - `SqliteClientError::ChainHeightUnknown` + - `SqliteClientError::CommitmentTree` + - `SqliteClientError::NonSequentialBlocks` - `zcash_client_backend::FsBlockDbError` has a new error variant: - `FsBlockDbError::CacheMiss` - `zcash_client_sqlite::FsBlockDb::write_block_metadata` now overwrites any @@ -46,6 +54,9 @@ and this library adheres to Rust's notion of previously performed by `init_blocks_table` is now handled by passing an `AccountBirthday` containing the note commitment tree frontier as of the end of the birthday height block to `create_account` instead. +- `zcash_client_sqlite::DataConnStmtCache` has been removed in favor of using + `rusqlite` caching for prepared statements. +- `zcash_client_sqlite::prepared` has been entirely removed. ### Fixed - Fixed an off-by-one error in the `BlockSource` implementation for the SQLite-backed diff --git a/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs b/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs index 582904359d..0d8c17b728 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/sapling_memo_consistency.rs @@ -46,10 +46,10 @@ impl RusqliteMigration for Migration

{ fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { let mut stmt_raw_tx = transaction.prepare( - "SELECT DISTINCT - transactions.id_tx, transactions.txid, + "SELECT DISTINCT + transactions.id_tx, transactions.txid, accounts.account, accounts.ufvk - FROM sent_notes + FROM sent_notes JOIN accounts ON sent_notes.from_account = accounts.account JOIN transactions ON transactions.id_tx = sent_notes.tx WHERE transactions.raw IS NOT NULL", @@ -78,8 +78,8 @@ impl RusqliteMigration for Migration

{ } let mut stmt_update_sent_memo = transaction.prepare( - "UPDATE sent_notes - SET memo = :memo + "UPDATE sent_notes + SET memo = :memo WHERE tx = :id_tx AND output_index = :output_index", )?; From 94c1f2413d1ae576899697df50dbd97d578fce82 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 8 Sep 2023 18:47:41 +0000 Subject: [PATCH 0161/1122] Migrate to `incrementalmerkletree 0.5`, `orchard 0.6` --- Cargo.toml | 5 ----- zcash_client_backend/CHANGELOG.md | 2 +- zcash_client_backend/Cargo.toml | 8 +++++--- zcash_client_sqlite/Cargo.toml | 10 ++++++---- zcash_primitives/CHANGELOG.md | 1 + zcash_primitives/Cargo.toml | 8 ++++---- zcash_proofs/CHANGELOG.md | 2 ++ zcash_proofs/Cargo.toml | 1 - 8 files changed, 19 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c673a73a40..044d879e93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,8 +17,3 @@ members = [ lto = true panic = 'abort' codegen-units = 1 - -[patch.crates-io] -incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } -shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "da97e6c399c5acddedad2c1730dbb7ee55499a2f" } -orchard = { git = "https://github.com/zcash/orchard.git", rev = "6ef89d5f154de2cf7b7dd87edb8d8c49158beebb" } diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 1ec9646004..46a033539f 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -58,7 +58,7 @@ and this library adheres to Rust's notion of ### Changed - MSRV is now 1.65.0. - Bumped dependencies to `hdwallet 0.4`, `zcash_primitives 0.13`, `zcash_note_encryption 0.4`, - `incrementalmerkletree 0.4`, `orchard 0.5`, `bs58 0.5`, `tempfile 3.5.0`, `prost 0.12`, + `incrementalmerkletree 0.5`, `orchard 0.6`, `bs58 0.5`, `tempfile 3.5.0`, `prost 0.12`, `tonic 0.10`. - `zcash_client_backend::data_api`: - `WalletRead::TxRef` has been removed in favor of consistently using `TxId` instead. diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 3757f3e2d7..f9ff95a559 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -20,8 +20,6 @@ exclude = ["*.proto"] development = ["zcash_proofs"] [dependencies] -incrementalmerkletree = { version = "0.4", features = ["legacy-api"] } -shardtree = "0.0" zcash_address = { version = "0.3", path = "../components/zcash_address" } zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_note_encryption = "0.4" @@ -56,7 +54,11 @@ subtle = "2.2.3" # - Shielded protocols bls12_381 = "0.8" group = "0.13" -orchard = { version = "0.5", default-features = false } +orchard = { version = "0.6", default-features = false } + +# - Note commitment trees +incrementalmerkletree = "0.5" +shardtree = "0.1" # - Test dependencies proptest = { version = "1.0.0", optional = true } diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index d1f31874ff..44b9f82156 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -14,8 +14,6 @@ edition = "2021" rust-version = "1.65" [dependencies] -incrementalmerkletree = { version = "0.4", features = ["legacy-api"] } -shardtree = { version = "0.0", features = ["legacy-api"] } zcash_client_backend = { version = "0.9", path = "../zcash_client_backend" } zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_primitives = { version = "0.12", path = "../zcash_primitives", default-features = false } @@ -38,6 +36,10 @@ jubjub = "0.10" # - Secret management secrecy = "0.8" +# - Note commitment trees +incrementalmerkletree = "0.5" +shardtree = { version = "0.1", features = ["legacy-api"] } + # - SQLite databases rusqlite = { version = "0.29.0", features = ["bundled", "time", "array"] } schemer = "0.2" @@ -51,8 +53,8 @@ maybe-rayon = {version = "0.1.0", default-features = false} [dev-dependencies] assert_matches = "1.5" -incrementalmerkletree = { version = "0.4", features = ["legacy-api", "test-dependencies"] } -shardtree = { version = "0.0", features = ["legacy-api", "test-dependencies"] } +incrementalmerkletree = { version = "0.5", features = ["test-dependencies"] } +shardtree = { version = "0.1", features = ["legacy-api", "test-dependencies"] } proptest = "1.0.0" rand_core = "0.6" regex = "1.4" diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 8ab995dea4..2c404d64fb 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -26,6 +26,7 @@ and this library adheres to Rust's notion of - `zcash_primitives::block::BlockHash::try_from_slice` ### Changed +- Migrated to `incrementalmerkletree 0.5`, `orchard 0.6`. - `zcash_primitives::transaction`: - `builder::Builder::{new, new_with_rng}` now take an optional `orchard_anchor` argument which must be provided in order to enable Orchard spends and recipients. diff --git a/zcash_primitives/Cargo.toml b/zcash_primitives/Cargo.toml index cec7ecc2c8..66489a31a4 100644 --- a/zcash_primitives/Cargo.toml +++ b/zcash_primitives/Cargo.toml @@ -44,10 +44,10 @@ ff = "0.13" group = { version = "0.13", features = ["wnaf-memuse"] } jubjub = "0.10" nonempty = "0.7" -orchard = { version = "0.5", default-features = false } +orchard = { version = "0.6", default-features = false } # - Note Commitment Trees -incrementalmerkletree = { version = "0.4", features = ["legacy-api"] } +incrementalmerkletree = { version = "0.5", features = ["legacy-api"] } # - Static constants lazy_static = "1" @@ -88,11 +88,11 @@ features = ["pre-zip-212"] [dev-dependencies] chacha20poly1305 = "0.10" criterion = "0.4" -incrementalmerkletree = { version = "0.4", features = ["legacy-api", "test-dependencies"] } +incrementalmerkletree = { version = "0.5", features = ["legacy-api", "test-dependencies"] } proptest = "1.0.0" assert_matches = "1.3.0" rand_xorshift = "0.3" -orchard = { version = "0.5", default-features = false, features = ["test-dependencies"] } +orchard = { version = "0.6", default-features = false, features = ["test-dependencies"] } [target.'cfg(unix)'.dev-dependencies] pprof = { version = "0.11", features = ["criterion", "flamegraph"] } # MSRV 1.56 diff --git a/zcash_proofs/CHANGELOG.md b/zcash_proofs/CHANGELOG.md index 339863d34c..b2cee4826e 100644 --- a/zcash_proofs/CHANGELOG.md +++ b/zcash_proofs/CHANGELOG.md @@ -6,6 +6,8 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Removed +- Unused `incrementalmerkletree` dependency. ## [0.12.1] - 2023-06-28 ### Changed diff --git a/zcash_proofs/Cargo.toml b/zcash_proofs/Cargo.toml index e110ba490e..52c6d08a90 100644 --- a/zcash_proofs/Cargo.toml +++ b/zcash_proofs/Cargo.toml @@ -24,7 +24,6 @@ zcash_primitives = { version = "0.12", path = "../zcash_primitives", default-fea bellman = { version = "0.14", default-features = false, features = ["groth16"] } bls12_381 = "0.8" group = "0.13" -incrementalmerkletree = { version = "0.4", features = ["legacy-api"] } jubjub = "0.10" lazy_static = "1" minreq = { version = "2", features = ["https"], optional = true } From 46cc6666df6202df3bbb7188f927143f676c4d98 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 8 Sep 2023 13:01:03 -0600 Subject: [PATCH 0162/1122] Address comments from code review. Co-authored-by: Daira Emma Hopwood --- zcash_client_backend/src/data_api.rs | 2 +- zcash_primitives/CHANGELOG.md | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index a8e83b6ee6..b20ced7aed 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -44,7 +44,7 @@ pub mod wallet; /// The height of subtree roots in the Sapling note commitment tree. /// -/// This conforms to the structure of subtree data returned by from +/// This conforms to the structure of subtree data returned by /// `lightwalletd` when using the `GetSubtreeRoots` GRPC call. pub const SAPLING_SHARD_HEIGHT: u8 = sapling::NOTE_COMMITMENT_TREE_DEPTH / 2; diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 2c404d64fb..331339a83f 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -41,8 +41,6 @@ and this library adheres to Rust's notion of `orchard::builder::InProgress` instead of `zcash_primitives::transaction::components::orchard::Unauthorized` - `zcash_primitives::consensus::NetworkUpgrade` now implements `PartialEq`, `Eq` -- `zcash_primitives::legacy::OpCode` now defines a complete set of transparent opcodes, - instead of just a minimal set. - `zcash_primitives::legacy::Script` now has a custom `Debug` implementation that renders script details in a much more legible fashion. - `zcash_primitives::sapling::redjubjub::Signature` now has a custom `Debug` From 67b84c25e06c0009b73d8c0afd91f28393cba4de Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 8 Sep 2023 19:45:54 +0000 Subject: [PATCH 0163/1122] zcash_primitives 0.13.0-rc.1 --- zcash_client_backend/Cargo.toml | 2 +- zcash_client_sqlite/CHANGELOG.md | 2 +- zcash_client_sqlite/Cargo.toml | 4 ++-- zcash_extensions/Cargo.toml | 2 +- zcash_primitives/CHANGELOG.md | 1 + zcash_primitives/Cargo.toml | 2 +- zcash_proofs/CHANGELOG.md | 3 +++ zcash_proofs/Cargo.toml | 2 +- 8 files changed, 11 insertions(+), 7 deletions(-) diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index f9ff95a559..c64638447d 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -23,7 +23,7 @@ development = ["zcash_proofs"] zcash_address = { version = "0.3", path = "../components/zcash_address" } zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_note_encryption = "0.4" -zcash_primitives = { version = "0.12", path = "../zcash_primitives", default-features = false } +zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false } # Dependencies exposed in a public API: # (Breaking upgrades to these require a breaking upgrade to this crate.) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 4aeedfc27a..ce020427b2 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -22,7 +22,7 @@ and this library adheres to Rust's notion of ### Changed - MSRV is now 1.65.0. - Bumped dependencies to `hdwallet 0.4`, `incrementalmerkletree 0.5`, `bs58 0.5`, - `zcash_primitives 0.12`, `rusqlite 0.29`, `schemer-rusqlite 0.2.2`, `time 0.3.22`, + `zcash_primitives 0.13`, `rusqlite 0.29`, `schemer-rusqlite 0.2.2`, `time 0.3.22`, `tempfile 3.5`, `zcash_note_encryption 0.4`, `zcash_proofs 0.12`, `prost 0.12` `zcash_client_backend 0.9`, `zcash_address 0.3` - Added dependencies on `shardtree 0.0`, `zcash_encoding 0.2`, `byteorder 1` diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 44b9f82156..8c860f171b 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -16,7 +16,7 @@ rust-version = "1.65" [dependencies] zcash_client_backend = { version = "0.9", path = "../zcash_client_backend" } zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } -zcash_primitives = { version = "0.12", path = "../zcash_primitives", default-features = false } +zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false } # Dependencies exposed in a public API: # (Breaking upgrades to these require a breaking upgrade to this crate.) @@ -61,7 +61,7 @@ regex = "1.4" tempfile = "3.5.0" zcash_note_encryption = "0.4" zcash_proofs = { version = "0.12", path = "../zcash_proofs" } -zcash_primitives = { version = "0.12", path = "../zcash_primitives", features = ["test-dependencies"] } +zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", features = ["test-dependencies"] } zcash_client_backend = { version = "0.9", path = "../zcash_client_backend", features = ["test-dependencies"] } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } diff --git a/zcash_extensions/Cargo.toml b/zcash_extensions/Cargo.toml index 912d78c7fa..1e6cbe0792 100644 --- a/zcash_extensions/Cargo.toml +++ b/zcash_extensions/Cargo.toml @@ -11,7 +11,7 @@ rust-version = "1.65" [dependencies] blake2b_simd = "1" -zcash_primitives = { version = "0.12", path = "../zcash_primitives", default-features = false, features = ["zfuture" ] } +zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false, features = ["zfuture" ] } [dev-dependencies] ff = "0.13" diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 331339a83f..0dd18961cf 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -7,6 +7,7 @@ and this library adheres to Rust's notion of ## [Unreleased] +## [0.13.0-rc.1] - 2023-09-08 ### Added - `zcash_primitives::consensus::BlockHeight::saturating_sub` - `zcash_primitives::transaction::builder`: diff --git a/zcash_primitives/Cargo.toml b/zcash_primitives/Cargo.toml index 66489a31a4..f438b8083b 100644 --- a/zcash_primitives/Cargo.toml +++ b/zcash_primitives/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_primitives" description = "Rust implementations of the Zcash primitives" -version = "0.12.0" +version = "0.13.0-rc.1" authors = [ "Jack Grigg ", "Kris Nuttycombe " diff --git a/zcash_proofs/CHANGELOG.md b/zcash_proofs/CHANGELOG.md index b2cee4826e..23c230a73a 100644 --- a/zcash_proofs/CHANGELOG.md +++ b/zcash_proofs/CHANGELOG.md @@ -6,6 +6,9 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Bumped dependencies to `zcash_primitives 0.13`. + ### Removed - Unused `incrementalmerkletree` dependency. diff --git a/zcash_proofs/Cargo.toml b/zcash_proofs/Cargo.toml index 52c6d08a90..fc2d72ebff 100644 --- a/zcash_proofs/Cargo.toml +++ b/zcash_proofs/Cargo.toml @@ -17,7 +17,7 @@ categories = ["cryptography::cryptocurrencies"] all-features = true [dependencies] -zcash_primitives = { version = "0.12", path = "../zcash_primitives", default-features = false } +zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false } # Dependencies exposed in a public API: # (Breaking upgrades to these require a breaking upgrade to this crate.) From 87ca71b8db5bc6859b3152fc4ebc82d26a7bcd73 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 8 Sep 2023 19:48:55 +0000 Subject: [PATCH 0164/1122] zcash_proofs 0.13.0-rc.1 --- zcash_client_backend/Cargo.toml | 2 +- zcash_client_sqlite/Cargo.toml | 2 +- zcash_extensions/Cargo.toml | 2 +- zcash_proofs/CHANGELOG.md | 2 ++ zcash_proofs/Cargo.toml | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index c64638447d..6b35373dff 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -88,7 +88,7 @@ proptest = "1.0.0" rand_core = "0.6" rand_xorshift = "0.3" tempfile = "3.5.0" -zcash_proofs = { version = "0.12", path = "../zcash_proofs", default-features = false } +zcash_proofs = { version = "=0.13.0-rc.1", path = "../zcash_proofs", default-features = false } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } time = ">=0.3.22, <0.3.24" # time 0.3.24 has MSRV 1.67 diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 8c860f171b..c620b4c56e 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -60,7 +60,7 @@ rand_core = "0.6" regex = "1.4" tempfile = "3.5.0" zcash_note_encryption = "0.4" -zcash_proofs = { version = "0.12", path = "../zcash_proofs" } +zcash_proofs = { version = "=0.13.0-rc.1", path = "../zcash_proofs" } zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", features = ["test-dependencies"] } zcash_client_backend = { version = "0.9", path = "../zcash_client_backend", features = ["test-dependencies"] } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } diff --git a/zcash_extensions/Cargo.toml b/zcash_extensions/Cargo.toml index 1e6cbe0792..4dfc014d13 100644 --- a/zcash_extensions/Cargo.toml +++ b/zcash_extensions/Cargo.toml @@ -18,7 +18,7 @@ ff = "0.13" jubjub = "0.10" rand_core = "0.6" zcash_address = { version = "0.3", path = "../components/zcash_address" } -zcash_proofs = { version = "0.12", path = "../zcash_proofs" } +zcash_proofs = { version = "=0.13.0-rc.1", path = "../zcash_proofs" } [features] transparent-inputs = [] diff --git a/zcash_proofs/CHANGELOG.md b/zcash_proofs/CHANGELOG.md index 23c230a73a..855a62df03 100644 --- a/zcash_proofs/CHANGELOG.md +++ b/zcash_proofs/CHANGELOG.md @@ -6,6 +6,8 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [0.13.0-rc.1] - 2023-09-08 ### Changed - Bumped dependencies to `zcash_primitives 0.13`. diff --git a/zcash_proofs/Cargo.toml b/zcash_proofs/Cargo.toml index fc2d72ebff..d62b5e546e 100644 --- a/zcash_proofs/Cargo.toml +++ b/zcash_proofs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_proofs" description = "Zcash zk-SNARK circuits and proving APIs" -version = "0.12.1" +version = "0.13.0-rc.1" authors = [ "Jack Grigg ", ] From 9db68c76eeccbe1e791b8e20dda5a9a6f8ebfca6 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 8 Sep 2023 19:50:53 +0000 Subject: [PATCH 0165/1122] zcash_client_backend 0.10.0-rc.1 --- zcash_client_backend/CHANGELOG.md | 1 + zcash_client_backend/Cargo.toml | 2 +- zcash_client_sqlite/CHANGELOG.md | 2 +- zcash_client_sqlite/Cargo.toml | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 46a033539f..a4f86265a1 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -7,6 +7,7 @@ and this library adheres to Rust's notion of ## [Unreleased] +## [0.10.0-rc.1] - 2023-09-08 ### Notable Changes - `zcash_client_backend` now supports out-of-order scanning of blockchain history. diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 6b35373dff..85f800a94c 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_backend" description = "APIs for creating shielded Zcash light clients" -version = "0.9.0" +version = "0.10.0-rc.1" authors = [ "Jack Grigg ", "Kris Nuttycombe " diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index ce020427b2..4a22639a89 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -24,7 +24,7 @@ and this library adheres to Rust's notion of - Bumped dependencies to `hdwallet 0.4`, `incrementalmerkletree 0.5`, `bs58 0.5`, `zcash_primitives 0.13`, `rusqlite 0.29`, `schemer-rusqlite 0.2.2`, `time 0.3.22`, `tempfile 3.5`, `zcash_note_encryption 0.4`, `zcash_proofs 0.12`, `prost 0.12` - `zcash_client_backend 0.9`, `zcash_address 0.3` + `zcash_client_backend 0.10`, `zcash_address 0.3`. - Added dependencies on `shardtree 0.0`, `zcash_encoding 0.2`, `byteorder 1` - A `CommitmentTree` variant has been added to `zcash_client_sqlite::wallet::init::WalletMigrationError` - `min_confirmations` parameter values are now more strongly enforced. Previously, diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index c620b4c56e..54a6897cb5 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -14,7 +14,7 @@ edition = "2021" rust-version = "1.65" [dependencies] -zcash_client_backend = { version = "0.9", path = "../zcash_client_backend" } +zcash_client_backend = { version = "=0.10.0-rc.1", path = "../zcash_client_backend" } zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false } @@ -62,7 +62,7 @@ tempfile = "3.5.0" zcash_note_encryption = "0.4" zcash_proofs = { version = "=0.13.0-rc.1", path = "../zcash_proofs" } zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", features = ["test-dependencies"] } -zcash_client_backend = { version = "0.9", path = "../zcash_client_backend", features = ["test-dependencies"] } +zcash_client_backend = { version = "=0.10.0-rc.1", path = "../zcash_client_backend", features = ["test-dependencies"] } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } [features] From 62e3299255dc66a2df314458d64bfabe4ba912e1 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 8 Sep 2023 19:52:43 +0000 Subject: [PATCH 0166/1122] zcash_client_sqlite 0.8.0-rc.1 --- zcash_client_sqlite/CHANGELOG.md | 8 +++++--- zcash_client_sqlite/Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 4a22639a89..ae91c2ae16 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -6,6 +6,8 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [0.8.0-rc.1] - 2023-09-08 ### Added - `zcash_client_sqlite::commitment_tree` Types related to management of note commitment trees using the `shardtree` crate. @@ -22,9 +24,9 @@ and this library adheres to Rust's notion of ### Changed - MSRV is now 1.65.0. - Bumped dependencies to `hdwallet 0.4`, `incrementalmerkletree 0.5`, `bs58 0.5`, - `zcash_primitives 0.13`, `rusqlite 0.29`, `schemer-rusqlite 0.2.2`, `time 0.3.22`, - `tempfile 3.5`, `zcash_note_encryption 0.4`, `zcash_proofs 0.12`, `prost 0.12` - `zcash_client_backend 0.10`, `zcash_address 0.3`. + `prost 0.12`, `rusqlite 0.29`, `schemer-rusqlite 0.2.2`, `time 0.3.22`, + `tempfile 3.5`, `zcash_address 0.3`, `zcash_note_encryption 0.4`, + `zcash_primitives 0.13`, `zcash_client_backend 0.10`. - Added dependencies on `shardtree 0.0`, `zcash_encoding 0.2`, `byteorder 1` - A `CommitmentTree` variant has been added to `zcash_client_sqlite::wallet::init::WalletMigrationError` - `min_confirmations` parameter values are now more strongly enforced. Previously, diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 54a6897cb5..5960847440 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_sqlite" description = "An SQLite-based Zcash light client" -version = "0.7.1" +version = "0.8.0-rc.1" authors = [ "Jack Grigg ", "Kris Nuttycombe " From fad1b913770cd97178499c46f106dee480c06e97 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 11 Sep 2023 20:21:01 +0000 Subject: [PATCH 0167/1122] Add `Cargo.lock` to repository This is currently "whatever lockfile happened to last work for str4d", but going forward will be the lockfile we use for testing our MSRV. See https://blog.rust-lang.org/2023/08/29/committing-lockfiles for rationale on this change. --- .gitignore | 1 - Cargo.lock | 3161 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 3161 insertions(+), 1 deletion(-) create mode 100644 Cargo.lock diff --git a/.gitignore b/.gitignore index fa8d85ac52..eb5a316cbd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ -Cargo.lock target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000000..62d1bedaf1 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3161 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.31", +] + +[[package]] +name = "async-trait" +version = "0.1.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.31", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" + +[[package]] +name = "base64ct" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a32fd6af2b5827bce66c29053ba0e7c42b9dcab01835835058558c10851a46b" + +[[package]] +name = "bech32" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" + +[[package]] +name = "bellman" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afceed28bac7f9f5a508bca8aeeff51cdfa4770c0b967ac55c621e2ddfd6171" +dependencies = [ + "bitvec", + "blake2s_simd", + "byteorder", + "crossbeam-channel", + "ff", + "group", + "lazy_static", + "log", + "num_cpus", + "pairing", + "rand_core", + "rayon", + "subtle", +] + +[[package]] +name = "bip0039" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef0f0152ec5cf17f49a5866afaa3439816207fd4f0a224c0211ffaf5e278426" +dependencies = [ + "hmac", + "pbkdf2", + "rand", + "sha2", + "unicode-normalization", + "zeroize", +] + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "blake2b_simd" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c2f0dc9a68c6317d884f97cc36cf5a3d20ba14ce404227df55e1af708ab04bc" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "blake2s_simd" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6637f448b9e61dfadbdcbae9a885fadee1f3eaffb1f8d3c1965d3ade8bdfd44f" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bls12_381" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7bc6d6292be3a19e6379786dac800f551e5865a5bb51ebbe3064ab80433f403" +dependencies = [ + "ff", + "group", + "pairing", + "rand_core", + "subtle", +] + +[[package]] +name = "bs58" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5353f36341f7451062466f0b755b96ac3a9547e4d7f6b70d603fc721a7d7896" +dependencies = [ + "sha2", + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + +[[package]] +name = "bytemuck" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + +[[package]] +name = "ciborium" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" + +[[package]] +name = "ciborium-ll" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", + "zeroize", +] + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "bitflags 1.3.2", + "clap_lex", + "indexmap 1.9.3", + "textwrap", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "constant_time_eq" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6" + +[[package]] +name = "cpp_demangle" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cpufeatures" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +dependencies = [ + "anes", + "atty", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.10.5", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "rand_core", + "typenum", +] + +[[package]] +name = "daggy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91a9304e55e9d601a39ae4deaba85406d5c0980e106f65afcf0460e9af1e7602" +dependencies = [ + "petgraph", +] + +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "equihash" +version = "0.2.0" +dependencies = [ + "blake2b_simd", + "byteorder", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "f4jumble" +version = "0.1.0" +dependencies = [ + "blake2b_simd", + "hex", + "proptest", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + +[[package]] +name = "ff" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" +dependencies = [ + "bitvec", + "rand_core", + "subtle", +] + +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + +[[package]] +name = "fixed-hash" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "fpe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c4b37de5ae15812a764c958297cfc50f5c010438f60c6ce75d11b802abd404" +dependencies = [ + "cbc", + "cipher", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "memuse", + "rand_core", + "subtle", +] + +[[package]] +name = "gumdrop" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bc700f989d2f6f0248546222d9b4258f5b02a171a431f8285a81c08142629e3" +dependencies = [ + "gumdrop_derive", +] + +[[package]] +name = "gumdrop_derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729f9bd3449d77e7831a18abfb7ba2f99ee813dfd15b8c2167c9a54ba20aa99d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "h2" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 1.9.3", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + +[[package]] +name = "halo2_gadgets" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126a150072b0c38c7b573fe3eaf0af944a7fed09e154071bf2436d3f016f7230" +dependencies = [ + "arrayvec", + "bitvec", + "ff", + "group", + "halo2_proofs", + "lazy_static", + "pasta_curves", + "rand", + "subtle", + "uint", +] + +[[package]] +name = "halo2_legacy_pdqsort" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47716fe1ae67969c5e0b2ef826f32db8c3be72be325e1aa3c1951d06b5575ec5" + +[[package]] +name = "halo2_proofs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b867a8d9bbb85fca76fff60652b5cd19b853a1c4d0665cb89bee68b18d2caf0" +dependencies = [ + "blake2b_simd", + "ff", + "group", + "halo2_legacy_pdqsort", + "maybe-rayon", + "pasta_curves", + "rand_core", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.0", +] + +[[package]] +name = "hdwallet" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a03ba7d4c9ea41552cd4351965ff96883e629693ae85005c501bb4b9e1c48a7" +dependencies = [ + "lazy_static", + "rand_core", + "ring", + "secp256k1", + "thiserror", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.9", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "incrementalmerkletree" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361c467824d4d9d4f284be4b2608800839419dccc4d4608f28345237fe354623" +dependencies = [ + "either", + "proptest", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + +[[package]] +name = "inferno" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73c0fefcb6d409a6587c07515951495d482006f89a21daa0f2f783aa4fd5e027" +dependencies = [ + "ahash", + "indexmap 2.0.0", + "is-terminal", + "itoa", + "log", + "num-format", + "once_cell", + "quick-xml", + "rgb", + "str_stack", +] + +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi 0.3.2", + "rustix", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "jubjub" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8499f7a74008aafbecb2a2e608a3e13e4dd3e84df198b604451efe93f2de6e61" +dependencies = [ + "bitvec", + "bls12_381", + "ff", + "group", + "rand_core", + "subtle", +] + +[[package]] +name = "known-folders" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b6f1427d9c43b1cce87434c4d9eca33f43bdbb6246a762aa823a582f74c1684" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin", +] + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "libm" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" + +[[package]] +name = "libsqlite3-sys" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" + +[[package]] +name = "lock_api" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "matchit" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" + +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + +[[package]] +name = "memchr" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" + +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "memuse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2145869435ace5ea6ea3d35f59be559317ec9a0d04e1812d5f185a87b6d36f1a" +dependencies = [ + "nonempty", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "minreq" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3de406eeb24aba36ed3829532fa01649129677186b44a49debec0ec574ca7da7" +dependencies = [ + "log", + "once_cell", + "rustls", + "webpki", + "webpki-roots", +] + +[[package]] +name = "mio" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nonempty" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" + +[[package]] +name = "num-bigint" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.2", + "libc", +] + +[[package]] +name = "object" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "orchard" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d31e68534df32024dcc89a8390ec6d7bef65edd87d91b45cfb481a2eb2d77c5" +dependencies = [ + "aes", + "bitvec", + "blake2b_simd", + "ff", + "fpe", + "group", + "halo2_gadgets", + "halo2_proofs", + "hex", + "incrementalmerkletree", + "lazy_static", + "memuse", + "nonempty", + "pasta_curves", + "proptest", + "rand", + "reddsa", + "serde", + "subtle", + "tracing", + "zcash_note_encryption 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "os_str_bytes" +version = "6.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" + +[[package]] +name = "pairing" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fec4625e73cf41ef4bb6846cafa6d44736525f442ba45e407c4a000a13996f" +dependencies = [ + "group", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "password-hash" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d791538a6dcc1e7cb7fe6f6b58aca40e7f79403c45b2bc274008b5e647af1d8" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + +[[package]] +name = "pasta_curves" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e57598f73cc7e1b2ac63c79c517b31a0877cd7c402cdcaa311b5208de7a095" +dependencies = [ + "blake2b_simd", + "ff", + "group", + "lazy_static", + "rand", + "static_assertions", + "subtle", +] + +[[package]] +name = "pbkdf2" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271779f35b581956db91a3e55737327a03aa051e90b1c47aeb189508533adfd7" +dependencies = [ + "digest", + "password-hash", +] + +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap 2.0.0", +] + +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.31", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "plotters" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" + +[[package]] +name = "plotters-svg" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "pprof" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196ded5d4be535690899a4631cc9f18cdc41b7ebf24a79400f46f48e49a11059" +dependencies = [ + "backtrace", + "cfg-if", + "criterion", + "findshlibs", + "inferno", + "libc", + "log", + "nix", + "once_cell", + "parking_lot", + "smallvec", + "symbolic-demangle", + "tempfile", + "thiserror", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "prettyplease" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +dependencies = [ + "proc-macro2", + "syn 2.0.31", +] + +[[package]] +name = "primitive-types" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f3486ccba82358b11a77516035647c34ba167dfa53312630de83b12bd4f3d66" +dependencies = [ + "fixed-hash", + "uint", +] + +[[package]] +name = "proc-macro2" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e35c06b98bf36aba164cc17cb25f7e232f5c4aeea73baa14b8a9f0d92dbfa65" +dependencies = [ + "bit-set", + "bitflags 1.3.2", + "byteorder", + "lazy_static", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax 0.6.29", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "prost" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8473a65b88506c106c28ae905ca4a2b83a2993640467a41bb3080627ddfd2c" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d3e647e9eb04ddfef78dfee2d5b3fefdf94821c84b710a3d8ebc89ede8b164" +dependencies = [ + "bytes", + "heck", + "itertools 0.11.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.31", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56075c27b20ae524d00f247b8a4dc333e5784f889fe63099f8e626bc8d73486c" +dependencies = [ + "anyhow", + "itertools 0.11.0", + "proc-macro2", + "quote", + "syn 2.0.31", +] + +[[package]] +name = "prost-types" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cebe0a918c97f86c217b0f76fd754e966f8b9f41595095cf7d74cb4e59d730f6" +dependencies = [ + "prost", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "reddsa" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78a5191930e84973293aa5f532b513404460cd2216c1cfb76d08748c15b40b02" +dependencies = [ + "blake2b_simd", + "byteorder", + "group", + "hex", + "jubjub", + "pasta_curves", + "rand_core", + "serde", + "thiserror", + "zeroize", +] + +[[package]] +name = "redjubjub" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a60db2c3bc9c6fd1e8631fee75abc008841d27144be744951d6b9b75f9b569c" +dependencies = [ + "rand_core", + "reddsa", + "serde", + "thiserror", + "zeroize", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax 0.7.5", +] + +[[package]] +name = "regex-automata" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.7.5", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + +[[package]] +name = "rgb" +version = "0.8.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20ec2d3e3fc7a92ced357df9cebd5a10b6fb2aa1ee797bf7e9ce2f17dffc8f59" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "ripemd" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f" +dependencies = [ + "digest", +] + +[[package]] +name = "rusqlite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" +dependencies = [ + "bitflags 2.4.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "time", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustix" +version = "0.38.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" +dependencies = [ + "bitflags 2.4.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustls" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schemer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835d8f9478fd2936195fc941a8666b0d0894d5bf3631cbb884a8ce8ba631f339" +dependencies = [ + "daggy", + "log", + "thiserror", + "uuid", +] + +[[package]] +name = "schemer-rusqlite" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fb5ac1fa52c58e2c6a618e3149d464e7ad8d0effca74990ea29c1fe2338b3b1" +dependencies = [ + "rusqlite", + "schemer", + "uuid", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "secp256k1" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4124a35fe33ae14259c490fd70fa199a32b9ce9502f2ee6bc4f81ec06fa65894" +dependencies = [ + "secp256k1-sys", +] + +[[package]] +name = "secp256k1-sys" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a129b9e9efbfb223753b9163c4ab3b13cff7fd9c7f010fbac25ab4099fa07e" +dependencies = [ + "cc", +] + +[[package]] +name = "secrecy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" +dependencies = [ + "zeroize", +] + +[[package]] +name = "serde" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.31", +] + +[[package]] +name = "serde_json" +version = "1.0.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shardtree" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c19f96dde3a8693874f7e7c53d95616569b4009379a903789efbd448f4ea9cc7" +dependencies = [ + "assert_matches", + "bitflags 2.4.0", + "either", + "incrementalmerkletree", + "proptest", + "tracing", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "str_stack" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "symbolic-common" +version = "10.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b55cdc318ede251d0957f07afe5fed912119b8c1bc5a7804151826db999e737" +dependencies = [ + "debugid", + "memmap2", + "stable_deref_trait", + "uuid", +] + +[[package]] +name = "symbolic-demangle" +version = "10.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79be897be8a483a81fff6a3a4e195b4ac838ef73ca42d348b3f722da9902e489" +dependencies = [ + "cpp_demangle", + "rustc-demangle", + "symbolic-common", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", +] + +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + +[[package]] +name = "thiserror" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.31", +] + +[[package]] +name = "time" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4" +dependencies = [ + "time-core", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2 0.5.3", + "windows-sys", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tonic" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5469afaf78a11265c343a88969045c1568aa8ecc6c787dbf756e92e70f199861" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b477abbe1d18c0b08f56cd01d1bc288668c5b5cfd19b2ae1886bbf599c546f1" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn 2.0.31", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.31", +] + +[[package]] +name = "tracing-core" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "uint" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "uuid" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wagyu-zcash-parameters" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c904628658374e651288f000934c33ef738b2d8b3e65d4100b70b395dbe2bb" +dependencies = [ + "wagyu-zcash-parameters-1", + "wagyu-zcash-parameters-2", + "wagyu-zcash-parameters-3", + "wagyu-zcash-parameters-4", + "wagyu-zcash-parameters-5", + "wagyu-zcash-parameters-6", +] + +[[package]] +name = "wagyu-zcash-parameters-1" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bf2e21bb027d3f8428c60d6a720b54a08bf6ce4e6f834ef8e0d38bb5695da8" + +[[package]] +name = "wagyu-zcash-parameters-2" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a616ab2e51e74cc48995d476e94de810fb16fc73815f390bf2941b046cc9ba2c" + +[[package]] +name = "wagyu-zcash-parameters-3" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14da1e2e958ff93c0830ee68e91884069253bf3462a67831b02b367be75d6147" + +[[package]] +name = "wagyu-zcash-parameters-4" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f058aeef03a2070e8666ffb5d1057d8bb10313b204a254a6e6103eb958e9a6d6" + +[[package]] +name = "wagyu-zcash-parameters-5" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ffe916b30e608c032ae1b734f02574a3e12ec19ab5cc5562208d679efe4969d" + +[[package]] +name = "wagyu-zcash-parameters-6" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7b6d5a78adc3e8f198e9cd730f219a695431467f7ec29dcfc63ade885feebe1" + +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.31", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.31", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0e74f82d49d545ad128049b7e88f6576df2da6b02e9ce565c6f533be576957e" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xdg" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213b7324336b53d2414b2db8537e56544d981803139155afa84f76eeebb7a546" + +[[package]] +name = "zcash_address" +version = "0.3.0" +dependencies = [ + "assert_matches", + "bech32", + "bs58", + "f4jumble", + "proptest", + "zcash_encoding", +] + +[[package]] +name = "zcash_client_backend" +version = "0.10.0-rc.1" +dependencies = [ + "assert_matches", + "base64", + "bech32", + "bls12_381", + "bs58", + "byteorder", + "crossbeam-channel", + "group", + "gumdrop", + "hdwallet", + "hex", + "incrementalmerkletree", + "jubjub", + "memuse", + "nom", + "orchard", + "percent-encoding", + "proptest", + "prost", + "rand_core", + "rand_xorshift", + "rayon", + "secrecy", + "shardtree", + "subtle", + "tempfile", + "time", + "tonic", + "tonic-build", + "tracing", + "which", + "zcash_address", + "zcash_encoding", + "zcash_note_encryption 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "zcash_primitives", + "zcash_proofs", +] + +[[package]] +name = "zcash_client_sqlite" +version = "0.8.0-rc.1" +dependencies = [ + "assert_matches", + "bs58", + "byteorder", + "group", + "hdwallet", + "incrementalmerkletree", + "jubjub", + "maybe-rayon", + "proptest", + "prost", + "rand_core", + "regex", + "rusqlite", + "schemer", + "schemer-rusqlite", + "secrecy", + "shardtree", + "tempfile", + "time", + "tracing", + "uuid", + "zcash_address", + "zcash_client_backend", + "zcash_encoding", + "zcash_note_encryption 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "zcash_primitives", + "zcash_proofs", +] + +[[package]] +name = "zcash_encoding" +version = "0.2.0" +dependencies = [ + "byteorder", + "nonempty", +] + +[[package]] +name = "zcash_extensions" +version = "0.0.0" +dependencies = [ + "blake2b_simd", + "ff", + "jubjub", + "rand_core", + "zcash_address", + "zcash_primitives", + "zcash_proofs", +] + +[[package]] +name = "zcash_history" +version = "0.3.0" +dependencies = [ + "assert_matches", + "blake2b_simd", + "byteorder", + "primitive-types", + "proptest", +] + +[[package]] +name = "zcash_note_encryption" +version = "0.4.0" +dependencies = [ + "chacha20", + "chacha20poly1305", + "cipher", + "rand_core", + "subtle", +] + +[[package]] +name = "zcash_note_encryption" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b4580cd6cee12e44421dac43169be8d23791650816bdb34e6ddfa70ac89c1c5" +dependencies = [ + "chacha20", + "chacha20poly1305", + "cipher", + "rand_core", + "subtle", +] + +[[package]] +name = "zcash_primitives" +version = "0.13.0-rc.1" +dependencies = [ + "aes", + "assert_matches", + "bip0039", + "bitvec", + "blake2b_simd", + "blake2s_simd", + "bls12_381", + "byteorder", + "chacha20poly1305", + "criterion", + "equihash", + "ff", + "fpe", + "group", + "hdwallet", + "hex", + "incrementalmerkletree", + "jubjub", + "lazy_static", + "memuse", + "nonempty", + "orchard", + "pprof", + "proptest", + "rand", + "rand_core", + "rand_xorshift", + "ripemd", + "secp256k1", + "sha2", + "subtle", + "zcash_address", + "zcash_encoding", + "zcash_note_encryption 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "zcash_proofs" +version = "0.13.0-rc.1" +dependencies = [ + "bellman", + "blake2b_simd", + "bls12_381", + "byteorder", + "criterion", + "group", + "home", + "jubjub", + "known-folders", + "lazy_static", + "minreq", + "pprof", + "rand_core", + "rand_xorshift", + "redjubjub", + "tracing", + "wagyu-zcash-parameters", + "xdg", + "zcash_primitives", +] + +[[package]] +name = "zeroize" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.31", +] From 15fed12e9539fd148d0f17a758cff89dce95acb8 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 11 Sep 2023 22:11:36 +0000 Subject: [PATCH 0168/1122] cargo update The `blake2*_simd` updates are ignored because they bump MSRV. --- Cargo.lock | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62d1bedaf1..e032351179 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,7 +114,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.32", ] [[package]] @@ -125,7 +125,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.32", ] [[package]] @@ -207,9 +207,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.3" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "base64ct" @@ -1243,9 +1243,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" [[package]] name = "lock_api" @@ -1604,7 +1604,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.32", ] [[package]] @@ -1699,7 +1699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.31", + "syn 2.0.32", ] [[package]] @@ -1768,7 +1768,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.31", + "syn 2.0.32", "tempfile", "which", ] @@ -1783,7 +1783,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.32", ] [[package]] @@ -2017,9 +2017,9 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.11" +version = "0.38.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" +checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" dependencies = [ "bitflags 2.4.0", "errno", @@ -2156,14 +2156,14 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.32", ] [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "2cc66a619ed80bf7a0f6b17dd063a84b88f6dea1813737cf469aef1d081142c2" dependencies = [ "itoa", "ryu", @@ -2222,9 +2222,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys", @@ -2296,9 +2296,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.31" +version = "2.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" dependencies = [ "proc-macro2", "quote", @@ -2353,7 +2353,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.32", ] [[package]] @@ -2419,7 +2419,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", - "socket2 0.5.3", + "socket2 0.5.4", "windows-sys", ] @@ -2495,7 +2495,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn 2.0.31", + "syn 2.0.32", ] [[package]] @@ -2550,7 +2550,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.32", ] [[package]] @@ -2746,7 +2746,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.32", "wasm-bindgen-shared", ] @@ -2768,7 +2768,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.32", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3157,5 +3157,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.32", ] From 92237a555881820e7002c887f0a4c467c743fead Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 11 Sep 2023 16:46:37 -0600 Subject: [PATCH 0169/1122] zcash_client_sqlite: `serialization` module should not have been public. --- zcash_client_sqlite/CHANGELOG.md | 2 -- zcash_client_sqlite/src/lib.rs | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index ae91c2ae16..ed00b29c52 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -11,8 +11,6 @@ and this library adheres to Rust's notion of ### Added - `zcash_client_sqlite::commitment_tree` Types related to management of note commitment trees using the `shardtree` crate. -- `zcash_client_sqlite::serialization` Serialization formats for data stored - as SQLite BLOBs in the wallet database. - A new default-enabled feature flag `multicore`. This allows users to disable multicore support by setting `default_features = false` on their `zcash_primitives`, `zcash_proofs`, and `zcash_client_sqlite` dependencies. diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 22f0ff8c68..35423a5cd9 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -85,7 +85,7 @@ use { pub mod chain; pub mod error; -pub mod serialization; +pub(crate) mod serialization; pub mod wallet; use wallet::{ From 28f1f7d296d64340a3cb2b3b4fefb1bd21b1bf91 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 11 Sep 2023 15:54:02 -0600 Subject: [PATCH 0170/1122] zcash_client_sqlite: Add a test for when gaps exist in checkpoints. --- zcash_client_sqlite/src/testing.rs | 4 ++ zcash_client_sqlite/src/wallet/sapling.rs | 66 +++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 9074792944..952c4fc973 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -170,6 +170,10 @@ where self.cache.block_source() } + pub(crate) fn latest_cached_block(&self) -> &Option<(BlockHeight, BlockHash, u32)> { + &self.latest_cached_block + } + /// Creates a fake block at the expected next height containing a single output of the /// given value, and inserts it into the cache. pub(crate) fn generate_next_block( diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index d29c92b813..f5fe3fe14c 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -1304,4 +1304,70 @@ pub(crate) mod tests { assert_eq!(spendable.len(), 1); } + + #[test] + fn checkpoint_gaps() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account, usk, birthday) = st.test_account().unwrap(); + let dfvk = st.test_account_sapling().unwrap(); + + // Generate a block that with funds belonging to our wallet. + st.generate_next_block( + &dfvk, + AddressType::DefaultExternal, + Amount::const_from_i64(500000), + ); + st.scan_cached_blocks(birthday.height(), 1); + + // Create a gap of 10 blocks having no shielded outputs, then add a block that doesn't + // belong to us so that we can get a checkpoint in the tree. + let not_our_key = ExtendedSpendingKey::master(&[]).to_diversifiable_full_viewing_key(); + let not_our_value = Amount::const_from_i64(10000); + st.generate_block_at( + birthday.height() + 10, + BlockHash([0; 32]), + ¬_our_key, + AddressType::DefaultExternal, + not_our_value, + st.latest_cached_block().unwrap().2, + ); + + // Scan the block + st.scan_cached_blocks(birthday.height() + 10, 1); + + // Fake that everything has been scanned + st.wallet() + .conn + .execute_batch("UPDATE scan_queue SET priority = 10") + .unwrap(); + + // Verify that our note is considered spendable + let spendable = select_spendable_sapling_notes( + &st.wallet().conn, + account, + Amount::const_from_i64(300000), + birthday.height() + 5, + &[], + ) + .unwrap(); + assert_eq!(spendable.len(), 1); + + // Attempt to spend the note with 5 confirmations + let to = not_our_key.default_address().1.into(); + assert_matches!( + st.create_spend_to_address( + &usk, + &to, + Amount::from_u64(10000).unwrap(), + None, + OvkPolicy::Sender, + NonZeroU32::new(5).unwrap(), + ), + Ok(_) + ); + } } From c1dc648cf0c6988136c894945f026f9bdbdfcc8e Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 11 Sep 2023 23:10:59 +0000 Subject: [PATCH 0171/1122] CI: Add job to build with latest Rust and dependencies This ensures that we catch non-MSRV compilation breakages caused by SemVer updates. --- .github/workflows/ci.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d03e3d29f..38f6b56d69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,25 @@ jobs: - name: Verify working directory is clean run: git diff --exit-code - build: + build-latest: + name: Latest build on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + id: toolchain + - run: rustup override set ${{steps.toolchain.outputs.name}} + - name: Remove lockfile to build with latest dependencies + run: rm Cargo.lock + - name: Build crates + run: cargo build --workspace --all-targets --all-features --verbose + - name: Verify working directory is clean (excluding lockfile) + run: git diff --exit-code ':!Cargo.lock' + + build-nodefault: name: Build target ${{ matrix.target }} runs-on: ubuntu-latest strategy: From 771e4013c3f7affcfc05f351cde4a2f9cf20058a Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 11 Sep 2023 17:02:42 -0600 Subject: [PATCH 0172/1122] Move shardtree serialization to the `zcash_client_backend` crate --- zcash_client_backend/Cargo.toml | 1 + zcash_client_backend/src/lib.rs | 3 +++ zcash_client_backend/src/serialization.rs | 1 + .../src/serialization/shardtree.rs | 0 zcash_client_sqlite/Cargo.toml | 2 +- zcash_client_sqlite/src/lib.rs | 1 - zcash_client_sqlite/src/wallet/commitment_tree.rs | 2 +- 7 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 zcash_client_backend/src/serialization.rs rename zcash_client_sqlite/src/serialization.rs => zcash_client_backend/src/serialization/shardtree.rs (100%) diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 85f800a94c..a56dc0c234 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -103,6 +103,7 @@ test-dependencies = [ "incrementalmerkletree/test-dependencies" ] unstable = ["byteorder"] +unstable-serialization = ["byteorder"] [lib] bench = false diff --git a/zcash_client_backend/src/lib.rs b/zcash_client_backend/src/lib.rs index 1cb87bc9f0..c2382ba874 100644 --- a/zcash_client_backend/src/lib.rs +++ b/zcash_client_backend/src/lib.rs @@ -20,6 +20,9 @@ pub mod scanning; pub mod wallet; pub mod zip321; +#[cfg(feature = "unstable-serialization")] +pub mod serialization; + pub use decrypt::{decrypt_transaction, DecryptedOutput, TransferType}; #[cfg(test)] diff --git a/zcash_client_backend/src/serialization.rs b/zcash_client_backend/src/serialization.rs new file mode 100644 index 0000000000..b11d1cb24e --- /dev/null +++ b/zcash_client_backend/src/serialization.rs @@ -0,0 +1 @@ +pub mod shardtree; diff --git a/zcash_client_sqlite/src/serialization.rs b/zcash_client_backend/src/serialization/shardtree.rs similarity index 100% rename from zcash_client_sqlite/src/serialization.rs rename to zcash_client_backend/src/serialization/shardtree.rs diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 5960847440..69974bb95d 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -62,7 +62,7 @@ tempfile = "3.5.0" zcash_note_encryption = "0.4" zcash_proofs = { version = "=0.13.0-rc.1", path = "../zcash_proofs" } zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", features = ["test-dependencies"] } -zcash_client_backend = { version = "=0.10.0-rc.1", path = "../zcash_client_backend", features = ["test-dependencies"] } +zcash_client_backend = { version = "=0.10.0-rc.1", path = "../zcash_client_backend", features = ["test-dependencies", "unstable-serialization"] } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } [features] diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 35423a5cd9..9d8734c076 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -85,7 +85,6 @@ use { pub mod chain; pub mod error; -pub(crate) mod serialization; pub mod wallet; use wallet::{ diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index f90b135bea..cbc6f750b8 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -18,7 +18,7 @@ use shardtree::{ use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer}; -use crate::serialization::{read_shard, write_shard}; +use zcash_client_backend::serialization::shardtree::{read_shard, write_shard}; /// Errors that can appear in SQLite-back [`ShardStore`] implementation operations. #[derive(Debug)] From 401af23484884c9fa8c9bb6713dedaf860be8570 Mon Sep 17 00:00:00 2001 From: Daira Emma Hopwood Date: Wed, 6 Sep 2023 19:50:30 +0100 Subject: [PATCH 0173/1122] Improve wallet tests. Signed-off-by: Daira Emma Hopwood --- zcash_client_sqlite/src/testing.rs | 74 ++++++++- zcash_client_sqlite/src/wallet/sapling.rs | 156 ++++++++++++------ zcash_primitives/CHANGELOG.md | 2 + .../src/transaction/components/amount.rs | 8 + 4 files changed, 191 insertions(+), 49 deletions(-) diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 9074792944..687c986cde 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -35,7 +35,7 @@ use zcash_client_backend::{ wallet::OvkPolicy, zip321, }; -use zcash_note_encryption::Domain; +use zcash_note_encryption::{Domain, COMPACT_NOTE_SIZE}; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight, Network, NetworkUpgrade, Parameters}, @@ -52,7 +52,7 @@ use zcash_primitives::{ Amount, }, fees::FeeRule, - TxId, + Transaction, TxId, }, zip32::{sapling::DiversifiableFullViewingKey, DiversifierIndex}, }; @@ -266,6 +266,34 @@ where (height, res) } + /// Creates a fake block at the expected next height containing only the given + /// transaction, and inserts it into the cache. + /// This assumes that the transaction only has Sapling spends and outputs. + /// + /// This generated block will be treated as the latest block, and subsequent calls to + /// [`Self::generate_next_block`] will build on it. + pub(crate) fn generate_next_block_from_tx( + &mut self, + tx: &Transaction, + ) -> (BlockHeight, Cache::InsertResult) { + let (height, prev_hash, initial_sapling_tree_size) = self + .latest_cached_block + .map(|(prev_height, prev_hash, end_size)| (prev_height + 1, prev_hash, end_size)) + .unwrap_or_else(|| (self.sapling_activation_height(), BlockHash([0; 32]), 0)); + + let cb = fake_compact_block_from_tx(height, prev_hash, tx, initial_sapling_tree_size); + let res = self.cache.insert(&cb); + + self.latest_cached_block = Some(( + height, + cb.hash(), + initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), + )); + + (height, res) + } + /// Invokes [`scan_cached_blocks`] with the given arguments, expecting success. pub(crate) fn scan_cached_blocks(&mut self, from_height: BlockHeight, limit: usize) { assert_matches!(self.try_scan_cached_blocks(from_height, limit), Ok(_)); @@ -663,6 +691,38 @@ pub(crate) fn fake_compact_block( (cb, note.nf(&dfvk.fvk().vk.nk, 0)) } +/// Create a fake CompactBlock at the given height containing only the given transaction. +/// This assumes that the transaction only has Sapling spends and outputs. +pub(crate) fn fake_compact_block_from_tx( + height: BlockHeight, + prev_hash: BlockHash, + tx: &Transaction, + initial_sapling_tree_size: u32, +) -> CompactBlock { + // Create a fake CompactTx + let mut ctx = CompactTx { + hash: tx.txid().as_ref().to_vec(), + ..Default::default() + }; + + if let Some(bundle) = tx.sapling_bundle() { + for spend in bundle.shielded_spends() { + ctx.spends.push(CompactSaplingSpend { + nf: spend.nullifier().to_vec(), + }); + } + for output in bundle.shielded_outputs() { + ctx.outputs.push(CompactSaplingOutput { + cmu: output.cmu().to_bytes().to_vec(), + ephemeral_key: output.ephemeral_key().0.to_vec(), + ciphertext: output.enc_ciphertext()[..COMPACT_NOTE_SIZE].to_vec(), + }); + } + } + + fake_compact_block_from_compact_tx(ctx, height, prev_hash, initial_sapling_tree_size) +} + /// Create a fake CompactBlock at the given height, spending a single note from the /// given address. #[allow(clippy::too_many_arguments)] @@ -737,6 +797,16 @@ pub(crate) fn fake_compact_block_spending( } }); + fake_compact_block_from_compact_tx(ctx, height, prev_hash, initial_sapling_tree_size) +} + +pub(crate) fn fake_compact_block_from_compact_tx( + ctx: CompactTx, + height: BlockHeight, + prev_hash: BlockHash, + initial_sapling_tree_size: u32, +) -> CompactBlock { + let mut rng = OsRng; let mut cb = CompactBlock { hash: { let mut hash = vec![0; 32]; diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index d29c92b813..d0cadc196c 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -512,8 +512,10 @@ pub(crate) mod tests { let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); st.scan_cached_blocks(h, 1); - // Verified balance matches total balance + // Spendable balance matches total balance assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); + assert_eq!( block_max_scanned(&st.wallet().conn) .unwrap() @@ -709,11 +711,16 @@ pub(crate) mod tests { let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); st.scan_cached_blocks(h1, 1); - // Verified balance matches total balance + // Spendable balance matches total balance at 1 confirmation. assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); - // Value is considered pending + // Value is considered pending at 10 confirmations. assert_eq!(st.get_pending_shielded_balance(account, 10), value); + assert_eq!( + st.get_spendable_balance(account, 10), + NonNegativeAmount::ZERO + ); // Wallet is fully scanned let summary = st.get_wallet_summary(1); @@ -766,7 +773,10 @@ pub(crate) mod tests { } st.scan_cached_blocks(h2 + 1, 8); - // Second spend still fails + // Total balance is value * number of blocks scanned (10). + assert_eq!(st.get_total_balance(account), (value * 10).unwrap()); + + // Spend still fails assert_matches!( st.create_spend_to_address( &usk, @@ -788,17 +798,38 @@ pub(crate) mod tests { let (h11, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); st.scan_cached_blocks(h11, 1); - // Second spend should now succeed - assert_matches!( - st.create_spend_to_address( + // Total balance is value * number of blocks scanned (11). + assert_eq!(st.get_total_balance(account), (value * 11).unwrap()); + // Spendable balance at 10 confirmations is value * 2. + assert_eq!(st.get_spendable_balance(account, 10), (value * 2).unwrap()); + assert_eq!( + st.get_pending_shielded_balance(account, 10), + (value * 9).unwrap() + ); + + // Spend should now succeed + let amount_sent = NonNegativeAmount::from_u64(70000).unwrap(); + let txid = st + .create_spend_to_address( &usk, &to, - Amount::from_u64(70000).unwrap(), + amount_sent.into(), None, OvkPolicy::Sender, NonZeroU32::new(10).unwrap(), - ), - Ok(_) + ) + .unwrap(); + let tx = &st.wallet().get_transaction(txid).unwrap(); + + let (h, _) = st.generate_next_block_from_tx(tx); + st.scan_cached_blocks(h, 1); + + // TODO: send to an account so that we can check its balance. + assert_eq!( + st.get_total_balance(account), + ((value * 11).unwrap() + - (amount_sent + NonNegativeAmount::from_u64(10000).unwrap()).unwrap()) + .unwrap() ); } @@ -816,9 +847,12 @@ pub(crate) mod tests { let value = NonNegativeAmount::from_u64(50000).unwrap(); let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); st.scan_cached_blocks(h1, 1); + + // Spendable balance matches total balance at 1 confirmation. assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); - // Send some of the funds to another address + // Send some of the funds to another address, but don't mine the tx. let extsk2 = ExtendedSpendingKey::master(&[]); let to = extsk2.default_address().1.into(); assert_matches!( @@ -886,16 +920,33 @@ pub(crate) mod tests { ); st.scan_cached_blocks(h43, 1); + // Spendable balance matches total balance at 1 confirmation. + assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); + // Second spend should now succeed - st.create_spend_to_address( - &usk, - &to, - Amount::from_u64(2000).unwrap(), - None, - OvkPolicy::Sender, - NonZeroU32::new(1).unwrap(), - ) - .unwrap(); + let amount_sent2 = NonNegativeAmount::from_u64(2000).unwrap(); + let txid2 = st + .create_spend_to_address( + &usk, + &to, + amount_sent2.into(), + None, + OvkPolicy::Sender, + NonZeroU32::new(1).unwrap(), + ) + .unwrap(); + let tx2 = &st.wallet().get_transaction(txid2).unwrap(); + + let (h, _) = st.generate_next_block_from_tx(tx2); + st.scan_cached_blocks(h, 1); + + // TODO: send to an account so that we can check its balance. + assert_eq!( + st.get_total_balance(account), + (value - (amount_sent2 + NonNegativeAmount::from_u64(10000).unwrap()).unwrap()) + .unwrap() + ); } #[test] @@ -912,7 +963,10 @@ pub(crate) mod tests { let value = NonNegativeAmount::from_u64(50000).unwrap(); let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); st.scan_cached_blocks(h1, 1); + + // Spendable balance matches total balance at 1 confirmation. assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); let extsk2 = ExtendedSpendingKey::master(&[]); let addr2 = extsk2.default_address().1; @@ -1007,16 +1061,15 @@ pub(crate) mod tests { let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note - let value = Amount::from_u64(60000).unwrap(); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let value = NonNegativeAmount::from_u64(60000).unwrap(); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); st.scan_cached_blocks(h, 1); - // Verified balance matches total balance - assert_eq!( - st.get_total_balance(account), - NonNegativeAmount::try_from(value).unwrap() - ); + // Spendable balance matches total balance at 1 confirmation. + assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); + // TODO: generate_next_block_from_tx does not currently support transparent outputs. let to = TransparentAddress::PublicKey([7; 20]).into(); assert_matches!( st.create_spend_to_address( @@ -1042,22 +1095,22 @@ pub(crate) mod tests { let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note owned by the internal spending key - let value = Amount::from_u64(60000).unwrap(); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::Internal, value); + let value = NonNegativeAmount::from_u64(60000).unwrap(); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::Internal, value.into()); st.scan_cached_blocks(h, 1); - // Verified balance matches total balance - assert_eq!( - st.get_total_balance(account), - NonNegativeAmount::try_from(value).unwrap() - ); + // Spendable balance matches total balance at 1 confirmation. + assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); - // the balance is considered pending + // Value is considered pending at 10 confirmations. + assert_eq!(st.get_pending_shielded_balance(account, 10), value); assert_eq!( - st.get_pending_shielded_balance(account, 10), - NonNegativeAmount::try_from(value).unwrap() + st.get_spendable_balance(account, 10), + NonNegativeAmount::ZERO ); + // TODO: generate_next_block_from_tx does not currently support transparent outputs. let to = TransparentAddress::PublicKey([7; 20]).into(); assert_matches!( st.create_spend_to_address( @@ -1100,12 +1153,10 @@ pub(crate) mod tests { st.scan_cached_blocks(h1, 11); - // Verified balance matches total balance - let total = Amount::from_u64(60000).unwrap(); - assert_eq!( - st.get_total_balance(account), - NonNegativeAmount::try_from(total).unwrap() - ); + // Spendable balance matches total balance + let total = NonNegativeAmount::from_u64(60000).unwrap(); + assert_eq!(st.get_total_balance(account), total); + assert_eq!(st.get_spendable_balance(account, 1), total); let input_selector = GreedyInputSelector::new( zip317::SingleOutputChangeStrategy::new(Zip317FeeRule::standard()), @@ -1148,15 +1199,26 @@ pub(crate) mod tests { }]) .unwrap(); - assert_matches!( - st.spend( + let txid = st + .spend( &input_selector, &usk, req, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), - ), - Ok(_) + ) + .unwrap(); + let tx = &st.wallet().get_transaction(txid).unwrap(); + + let (h, _) = st.generate_next_block_from_tx(tx); + st.scan_cached_blocks(h, 1); + + // TODO: send to an account so that we can check its balance. + // We sent back to the same account so the amount_sent should be included + // in the total balance. + assert_eq!( + st.get_total_balance(account), + (total - NonNegativeAmount::from_u64(10000).unwrap()).unwrap() ); } diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 0dd18961cf..e708c8f36d 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -6,6 +6,8 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Trait implementation `Mul` for `NonNegativeAmount`. ## [0.13.0-rc.1] - 2023-09-08 ### Added diff --git a/zcash_primitives/src/transaction/components/amount.rs b/zcash_primitives/src/transaction/components/amount.rs index dbb3e85d7c..4c03189629 100644 --- a/zcash_primitives/src/transaction/components/amount.rs +++ b/zcash_primitives/src/transaction/components/amount.rs @@ -306,6 +306,14 @@ impl Sub for Option { } } +impl Mul for NonNegativeAmount { + type Output = Option; + + fn mul(self, rhs: usize) -> Option { + (self.0 * rhs).map(NonNegativeAmount) + } +} + /// A type for balance violations in amount addition and subtraction /// (overflow and underflow of allowed ranges) #[derive(Copy, Clone, Debug, PartialEq, Eq)] From 101c9c0ea491853b67dda9fdfbf8ae35948df202 Mon Sep 17 00:00:00 2001 From: Daira Emma Hopwood Date: Mon, 11 Sep 2023 17:21:09 +0100 Subject: [PATCH 0174/1122] Add passing test for current behaviour of wallet restore from seed. refs #936 Signed-off-by: Daira Emma Hopwood --- zcash_client_sqlite/src/testing.rs | 32 ++++++ zcash_client_sqlite/src/wallet/sapling.rs | 124 +++++++++++++++++++++- 2 files changed, 155 insertions(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 687c986cde..353700371a 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -319,6 +319,38 @@ where limit, ) } + + /// Resets the wallet using a new wallet database but with the same cache of blocks, + /// and returns the old wallet database file. + /// + /// This does not recreate accounts, nor does it rescan the cached blocks. + /// The resulting wallet has no test account. + /// Before using any `generate_*` method on the reset state, call `reset_latest_cached_block()`. + pub(crate) fn reset(&mut self) -> NamedTempFile { + let network = self.network(); + self.latest_cached_block = None; + let tf = std::mem::replace(&mut self._data_file, NamedTempFile::new().unwrap()); + self.db_data = WalletDb::for_path(self._data_file.path(), network).unwrap(); + self.test_account = None; + init_wallet_db(&mut self.db_data, None).unwrap(); + tf + } + + /// Reset the latest cached block to the most recent one in the cache database. + #[allow(dead_code)] + pub(crate) fn reset_latest_cached_block(&mut self) { + self.cache + .block_source() + .with_blocks::<_, Infallible>(None, None, |block: CompactBlock| { + self.latest_cached_block = Some(( + BlockHeight::from_u32(block.height.try_into().unwrap()), + BlockHash::from_slice(block.hash.as_slice()), + block.chain_metadata.unwrap().sapling_commitment_tree_size, + )); + Ok(()) + }) + .unwrap(); + } } impl TestState { diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index d0cadc196c..a600dda5d2 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -434,6 +434,7 @@ pub(crate) mod tests { use std::{convert::Infallible, num::NonZeroU32}; use incrementalmerkletree::Hashable; + use secrecy::Secret; use zcash_proofs::prover::LocalTxProver; use zcash_primitives::{ @@ -464,6 +465,7 @@ pub(crate) mod tests { error::Error, wallet::input_selection::{GreedyInputSelector, GreedyInputSelectorError}, AccountBirthday, Ratio, ShieldedProtocol, WalletCommitmentTrees, WalletRead, + WalletWrite, }, decrypt_transaction, fees::{fixed, zip317, DustOutputPolicy}, @@ -484,7 +486,7 @@ pub(crate) mod tests { #[cfg(feature = "transparent-inputs")] use { - zcash_client_backend::{data_api::WalletWrite, wallet::WalletTransparentOutput}, + zcash_client_backend::wallet::WalletTransparentOutput, zcash_primitives::transaction::components::{OutPoint, TxOut}, }; @@ -1125,6 +1127,126 @@ pub(crate) mod tests { ); } + #[test] + fn external_address_change_spends_detected_in_restore_from_seed() { + let mut st = TestBuilder::new().with_block_cache().build(); + + // Add two accounts to the wallet. + let seed = Secret::new([0u8; 32].to_vec()); + let birthday = AccountBirthday::from_sapling_activation(&st.network()); + let (_, usk) = st + .wallet_mut() + .create_account(&seed, birthday.clone()) + .unwrap(); + let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); + + let (_, usk2) = st + .wallet_mut() + .create_account(&seed, birthday.clone()) + .unwrap(); + let dfvk2 = usk2.sapling().to_diversifiable_full_viewing_key(); + + // Add funds to the wallet in a single note + let value = NonNegativeAmount::from_u64(100000).unwrap(); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + st.scan_cached_blocks(h, 1); + + // Spendable balance matches total balance + assert_eq!(st.get_total_balance(AccountId::from(0)), value); + assert_eq!(st.get_spendable_balance(AccountId::from(0), 1), value); + assert_eq!( + st.get_total_balance(AccountId::from(1)), + NonNegativeAmount::ZERO + ); + + let amount_sent = NonNegativeAmount::from_u64(20000).unwrap(); + let amount_legacy_change = NonNegativeAmount::from_u64(30000).unwrap(); + let addr = dfvk.default_address().1; + let addr2 = dfvk2.default_address().1; + let req = TransactionRequest::new(vec![ + // payment to an external recipient + Payment { + recipient_address: RecipientAddress::Shielded(addr2), + amount: amount_sent.into(), + memo: None, + label: None, + message: None, + other_params: vec![], + }, + // payment back to the originating wallet, simulating legacy change + Payment { + recipient_address: RecipientAddress::Shielded(addr), + amount: amount_legacy_change.into(), + memo: None, + label: None, + message: None, + other_params: vec![], + }, + ]) + .unwrap(); + + let fee_rule = FixedFeeRule::standard(); + let input_selector = GreedyInputSelector::new( + fixed::SingleOutputChangeStrategy::new(fee_rule), + DustOutputPolicy::default(), + ); + + let txid = st + .spend( + &input_selector, + &usk, + req, + OvkPolicy::Sender, + NonZeroU32::new(1).unwrap(), + ) + .unwrap(); + let tx = &st.wallet().get_transaction(txid).unwrap(); + + let amount_left = + (value - (amount_sent + fee_rule.fixed_fee().try_into().unwrap()).unwrap()).unwrap(); + let pending_change = (amount_left - amount_legacy_change).unwrap(); + + // The "legacy change" is not counted by get_pending_change(). + assert_eq!(st.get_pending_change(AccountId::from(0), 1), pending_change); + // We spent the only note so we only have pending change. + assert_eq!(st.get_total_balance(AccountId::from(0)), pending_change); + + let (h, _) = st.generate_next_block_from_tx(tx); + st.scan_cached_blocks(h, 1); + + assert_eq!(st.get_total_balance(AccountId::from(1)), amount_sent); + assert_eq!(st.get_total_balance(AccountId::from(0)), amount_left); + + st.reset(); + + // Account creation and DFVK derivation should be deterministic. + let (_, restored_usk) = st + .wallet_mut() + .create_account(&seed, birthday.clone()) + .unwrap(); + assert_eq!( + restored_usk + .sapling() + .to_diversifiable_full_viewing_key() + .to_bytes(), + dfvk.to_bytes() + ); + + let (_, restored_usk2) = st.wallet_mut().create_account(&seed, birthday).unwrap(); + assert_eq!( + restored_usk2 + .sapling() + .to_diversifiable_full_viewing_key() + .to_bytes(), + dfvk2.to_bytes() + ); + + st.scan_cached_blocks(st.sapling_activation_height(), 2); + + assert_eq!(st.get_total_balance(AccountId::from(1)), amount_sent); + assert_eq!(st.get_total_balance(AccountId::from(0)), amount_left); + } + #[test] fn zip317_spend() { let mut st = TestBuilder::new() From 24e8c82546b5ac3d65201c6b915118e98613debe Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 11 Sep 2023 15:18:17 -0600 Subject: [PATCH 0175/1122] zcash_client_backend: Get commitment tree depth for a given number of confirmations from the database. This fixes the following bug: Due to complexities related to non-linear scanning, checkpoints are only added to the wallet's commitment tree in cases where there are notes discovered within a scanned block. At present, the `shardtree` API only makes it possible to add multiple checkpoints of the same tree state when adding checkpoints at the chain tip, and this functionality is not used by `zcash_client_backend` because we perform checkpoint insertion in batches that may contain gaps in the case that multiple blocks contain no Sapling notes. While it would be possible to fix this by altering the `shardtree` API to permit explicit insertion of multiple checkpoints of the same tree state at a given note position, this fix takes a simpler approach. Instead of ensuring that a checkpoint exists at every block and computing the required checkpoint depth directly from the minimum number of confirmations required when attempting a spend, we alter the `WalletCommitmentTrees` API to allow internal information of the note commitment tree to be used to determine this checkpoint depth, given the minimum number of commitments as an argument. This allows us to select a usable checkpoint from the sparse checkpoint set that resulted from the sparse insertion of checkpoints described above. --- zcash_client_backend/src/data_api.rs | 17 ++++++++++ zcash_client_backend/src/data_api/wallet.rs | 4 ++- zcash_client_sqlite/src/lib.rs | 28 +++++++++++++-- .../src/wallet/commitment_tree.rs | 34 +++++++++++++++++-- 4 files changed, 78 insertions(+), 5 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index b20ced7aed..098ad9e61e 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -867,6 +867,16 @@ pub trait WalletCommitmentTrees { Error = Self::Error, >; + /// Returns the depth of the checkpoint in the tree that can be used to create a witness at the + /// anchor having the given number of confirmations. + /// + /// This assumes that at any time a note is added to the tree, a checkpoint is created for the + /// end of the block in which that note was discovered. + fn get_checkpoint_depth( + &self, + min_confirmations: NonZeroU32, + ) -> Result>; + fn with_sapling_tree_mut(&mut self, callback: F) -> Result where for<'a> F: FnMut( @@ -1182,5 +1192,12 @@ pub mod testing { Ok(()) } + + fn get_checkpoint_depth( + &self, + min_confirmations: NonZeroU32, + ) -> Result> { + Ok(usize::try_from(u32::from(min_confirmations) - 1).unwrap()) + } } } diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index da820e7298..71d0503694 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -488,6 +488,8 @@ where // are no possible transparent inputs, so we ignore those let mut builder = Builder::new(params.clone(), proposal.min_target_height(), None); + let checkpoint_depth = wallet_db.get_checkpoint_depth(min_confirmations)?; + wallet_db.with_sapling_tree_mut::<_, _, Error<_, _, _, _, _>>(|sapling_tree| { for selected in proposal.sapling_inputs() { let (note, key, merkle_path) = select_key_for_note( @@ -495,7 +497,7 @@ where selected, usk.sapling(), &dfvk, - usize::try_from(u32::from(min_confirmations) - 1).unwrap(), + checkpoint_depth, )? .ok_or(Error::NoteMismatch(selected.note_id))?; diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 22f0ff8c68..3c30f3f6b2 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -44,7 +44,10 @@ use std::{ }; use incrementalmerkletree::Position; -use shardtree::{error::ShardTreeError, ShardTree}; +use shardtree::{ + error::{QueryError, ShardTreeError}, + ShardTree, +}; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight}, @@ -89,7 +92,7 @@ pub mod serialization; pub mod wallet; use wallet::{ - commitment_tree::{self, put_shard_roots}, + commitment_tree::{self, get_checkpoint_depth, put_shard_roots}, SubtreeScanProgress, }; @@ -795,6 +798,15 @@ impl WalletCommitmentTrees for WalletDb Result> { + get_checkpoint_depth(&self.conn, SAPLING_TABLES_PREFIX, min_confirmations) + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))? + .ok_or(ShardTreeError::Query(QueryError::CheckpointPruned)) + } } impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb, P> { @@ -835,6 +847,18 @@ impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb Result> { + get_checkpoint_depth(self.conn.0, SAPLING_TABLES_PREFIX, min_confirmations) + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))? + // `CheckpointPruned` is perhaps a little misleading; in this case it's that + // the chain tip is unknown, but if that were the case we should never have been + // calling this anyway. + .ok_or(ShardTreeError::Query(QueryError::CheckpointPruned)) + } } /// A handle for the SQLite block source. diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index f90b135bea..f05ab043c4 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -4,6 +4,7 @@ use std::{ error, fmt, io::{self, Cursor}, marker::PhantomData, + num::NonZeroU32, ops::Range, sync::Arc, }; @@ -20,6 +21,8 @@ use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer}; use crate::serialization::{read_shard, write_shard}; +use super::scan_queue_extrema; + /// Errors that can appear in SQLite-back [`ShardStore`] implementation operations. #[derive(Debug)] pub enum Error { @@ -724,8 +727,8 @@ pub(crate) fn get_checkpoint( .query_row( &format!( "SELECT position - FROM {}_tree_checkpoints - WHERE checkpoint_id = ?", + FROM {}_tree_checkpoints + WHERE checkpoint_id = ?", table_prefix ), [u32::from(checkpoint_id)], @@ -747,6 +750,33 @@ pub(crate) fn get_checkpoint( .transpose() } +pub(crate) fn get_checkpoint_depth( + conn: &rusqlite::Connection, + table_prefix: &'static str, + min_confirmations: NonZeroU32, +) -> Result, rusqlite::Error> { + scan_queue_extrema(conn)? + .map(|(_, max)| max) + .map(|chain_tip| { + let max_checkpoint_height = + u32::from(chain_tip).saturating_sub(u32::from(min_confirmations) - 1); + + // We exclude from consideration all checkpoints having heights greater than the maximum + // checkpoint height. The checkpoint depth is the number of excluded checkpoints + 1. + conn.query_row( + &format!( + "SELECT COUNT(*) + FROM {}_tree_checkpoints + WHERE checkpoint_id > :max_checkpoint_height", + table_prefix + ), + named_params![":max_checkpoint_height": max_checkpoint_height], + |row| row.get::<_, usize>(0).map(|s| s + 1), + ) + }) + .transpose() +} + pub(crate) fn get_checkpoint_at_depth( conn: &rusqlite::Connection, table_prefix: &'static str, From f2dcba3a34bb5f00ec670f478029621201529b2d Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 11 Sep 2023 17:04:49 -0600 Subject: [PATCH 0176/1122] Apply suggestions from code review Co-authored-by: str4d --- zcash_client_backend/src/data_api.rs | 2 +- zcash_client_sqlite/src/lib.rs | 3 +++ zcash_client_sqlite/src/wallet/sapling.rs | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 098ad9e61e..86cf4c4e33 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -1197,7 +1197,7 @@ pub mod testing { &self, min_confirmations: NonZeroU32, ) -> Result> { - Ok(usize::try_from(u32::from(min_confirmations) - 1).unwrap()) + Ok(usize::try_from(u32::from(min_confirmations)).unwrap()) } } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 3c30f3f6b2..24fe26b301 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -805,6 +805,9 @@ impl WalletCommitmentTrees for WalletDb Result> { get_checkpoint_depth(&self.conn, SAPLING_TABLES_PREFIX, min_confirmations) .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))? + // `CheckpointPruned` is perhaps a little misleading; in this case it's that + // the chain tip is unknown, but if that were the case we should never have been + // calling this anyway. .ok_or(ShardTreeError::Query(QueryError::CheckpointPruned)) } } diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index f5fe3fe14c..dc6b6ff9e2 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -1315,7 +1315,7 @@ pub(crate) mod tests { let (account, usk, birthday) = st.test_account().unwrap(); let dfvk = st.test_account_sapling().unwrap(); - // Generate a block that with funds belonging to our wallet. + // Generate a block with funds belonging to our wallet. st.generate_next_block( &dfvk, AddressType::DefaultExternal, From 24068cd63ad599492cddcbec8f999454feba1d17 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 12 Sep 2023 08:31:03 -0600 Subject: [PATCH 0177/1122] zcash_client_backend 0.10.0-rc.2 --- Cargo.lock | 2 +- zcash_client_backend/Cargo.toml | 2 +- zcash_client_sqlite/Cargo.toml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e032351179..c1ee378614 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2946,7 +2946,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" -version = "0.10.0-rc.1" +version = "0.10.0-rc.2" dependencies = [ "assert_matches", "base64", diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index a56dc0c234..d6ac7701e7 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_backend" description = "APIs for creating shielded Zcash light clients" -version = "0.10.0-rc.1" +version = "0.10.0-rc.2" authors = [ "Jack Grigg ", "Kris Nuttycombe " diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 69974bb95d..d38ba1c2f7 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -14,7 +14,7 @@ edition = "2021" rust-version = "1.65" [dependencies] -zcash_client_backend = { version = "=0.10.0-rc.1", path = "../zcash_client_backend" } +zcash_client_backend = { version = "=0.10.0-rc.2", path = "../zcash_client_backend" } zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false } @@ -62,7 +62,7 @@ tempfile = "3.5.0" zcash_note_encryption = "0.4" zcash_proofs = { version = "=0.13.0-rc.1", path = "../zcash_proofs" } zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", features = ["test-dependencies"] } -zcash_client_backend = { version = "=0.10.0-rc.1", path = "../zcash_client_backend", features = ["test-dependencies", "unstable-serialization"] } +zcash_client_backend = { version = "=0.10.0-rc.2", path = "../zcash_client_backend", features = ["test-dependencies", "unstable-serialization"] } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } [features] From e6b34eeaa2861a7fb3e7f6cdc1748481877e6fff Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 12 Sep 2023 08:32:34 -0600 Subject: [PATCH 0178/1122] zcash_client_sqlite 0.8.0-rc.2 --- Cargo.lock | 2 +- zcash_client_sqlite/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1ee378614..dc4ed9fc5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2988,7 +2988,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" -version = "0.8.0-rc.1" +version = "0.8.0-rc.2" dependencies = [ "assert_matches", "bs58", diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index d38ba1c2f7..be10f848e3 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_sqlite" description = "An SQLite-based Zcash light client" -version = "0.8.0-rc.1" +version = "0.8.0-rc.2" authors = [ "Jack Grigg ", "Kris Nuttycombe " From b32aa3d2a6c51df328031d5f8544bd93465f733a Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 12 Sep 2023 09:33:11 -0600 Subject: [PATCH 0179/1122] Fix missing zcash_client_backend feature dependency. --- zcash_client_sqlite/CHANGELOG.md | 2 +- zcash_client_sqlite/Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index ed00b29c52..2e1c9533d4 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -7,7 +7,7 @@ and this library adheres to Rust's notion of ## [Unreleased] -## [0.8.0-rc.1] - 2023-09-08 +## [0.8.0-rc.3] - 2023-09-12 ### Added - `zcash_client_sqlite::commitment_tree` Types related to management of note commitment trees using the `shardtree` crate. diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index be10f848e3..fba2c8f3fe 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_sqlite" description = "An SQLite-based Zcash light client" -version = "0.8.0-rc.2" +version = "0.8.0-rc.3" authors = [ "Jack Grigg ", "Kris Nuttycombe " @@ -14,7 +14,7 @@ edition = "2021" rust-version = "1.65" [dependencies] -zcash_client_backend = { version = "=0.10.0-rc.2", path = "../zcash_client_backend" } +zcash_client_backend = { version = "=0.10.0-rc.2", path = "../zcash_client_backend", features = ["unstable-serialization"]} zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false } From 45ced4d16433652d47e6690c9af00a5943755cec Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 12 Sep 2023 12:11:46 -0600 Subject: [PATCH 0180/1122] Update Cargo.lock --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index dc4ed9fc5d..4547bfefad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2988,7 +2988,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" -version = "0.8.0-rc.2" +version = "0.8.0-rc.3" dependencies = [ "assert_matches", "bs58", From 1575f2db88f288b91a91bf64d46f89f80ac34859 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 15 Sep 2023 13:37:10 -0600 Subject: [PATCH 0181/1122] zcash_client_backend: make the `SpanningTree` type usable outside of `zcash_client_sqlite` This adds the `data_api::scanning::spanning_tree` module under a new `unstable-spanning-tree` feature flag, making it available to other implementations who want to be able to write their own storage backends without having to reinvent the spanning tree logic. --- zcash_client_backend/Cargo.toml | 1 + zcash_client_backend/src/data_api/scanning.rs | 3 + .../src/data_api/scanning/spanning_tree.rs | 811 +++++++++++++++++ zcash_client_sqlite/Cargo.toml | 2 +- zcash_client_sqlite/src/wallet/scanning.rs | 820 +----------------- 5 files changed, 833 insertions(+), 804 deletions(-) create mode 100644 zcash_client_backend/src/data_api/scanning/spanning_tree.rs diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index d6ac7701e7..c38f851975 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -104,6 +104,7 @@ test-dependencies = [ ] unstable = ["byteorder"] unstable-serialization = ["byteorder"] +unstable-spanning-tree = [] [lib] bench = false diff --git a/zcash_client_backend/src/data_api/scanning.rs b/zcash_client_backend/src/data_api/scanning.rs index dcd9e0ff69..9f978025a1 100644 --- a/zcash_client_backend/src/data_api/scanning.rs +++ b/zcash_client_backend/src/data_api/scanning.rs @@ -3,6 +3,9 @@ use std::ops::Range; use zcash_primitives::consensus::BlockHeight; +#[cfg(feature = "unstable-spanning-tree")] +pub mod spanning_tree; + /// Scanning range priority levels. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum ScanPriority { diff --git a/zcash_client_backend/src/data_api/scanning/spanning_tree.rs b/zcash_client_backend/src/data_api/scanning/spanning_tree.rs new file mode 100644 index 0000000000..58631de8fb --- /dev/null +++ b/zcash_client_backend/src/data_api/scanning/spanning_tree.rs @@ -0,0 +1,811 @@ +use std::cmp::{max, Ordering}; +use std::ops::{Not, Range}; + +use zcash_primitives::consensus::BlockHeight; + +use super::{ScanPriority, ScanRange}; + +#[derive(Debug, Clone, Copy)] +enum InsertOn { + Left, + Right, +} + +struct Insert { + on: InsertOn, + force_rescan: bool, +} + +impl Insert { + fn left(force_rescan: bool) -> Self { + Insert { + on: InsertOn::Left, + force_rescan, + } + } + + fn right(force_rescan: bool) -> Self { + Insert { + on: InsertOn::Right, + force_rescan, + } + } +} + +impl Not for Insert { + type Output = Self; + + fn not(self) -> Self::Output { + Insert { + on: match self.on { + InsertOn::Left => InsertOn::Right, + InsertOn::Right => InsertOn::Left, + }, + force_rescan: self.force_rescan, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Dominance { + Left, + Right, + Equal, +} + +impl From for Dominance { + fn from(value: Insert) -> Self { + match value.on { + InsertOn::Left => Dominance::Left, + InsertOn::Right => Dominance::Right, + } + } +} + +// This implements the dominance rule for range priority. If the inserted range's priority is +// `Verify`, this replaces any existing priority. Otherwise, if the current priority is +// `Scanned`, it remains as `Scanned`; and if the new priority is `Scanned`, it +// overrides any existing priority. +fn dominance(current: &ScanPriority, inserted: &ScanPriority, insert: Insert) -> Dominance { + match (current.cmp(inserted), (current, inserted)) { + (Ordering::Equal, _) => Dominance::Equal, + (_, (_, ScanPriority::Verify | ScanPriority::Scanned)) => Dominance::from(insert), + (_, (ScanPriority::Scanned, _)) if !insert.force_rescan => Dominance::from(!insert), + (Ordering::Less, _) => Dominance::from(insert), + (Ordering::Greater, _) => Dominance::from(!insert), + } +} + +/// In the comments for each alternative, `()` represents the left range and `[]` represents the right range. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RangeOrdering { + /// `( ) [ ]` + LeftFirstDisjoint, + /// `( [ ) ]` + LeftFirstOverlap, + /// `[ ( ) ]` + LeftContained, + /// ```text + /// ( ) + /// [ ] + /// ``` + Equal, + /// `( [ ] )` + RightContained, + /// `[ ( ] )` + RightFirstOverlap, + /// `[ ] ( )` + RightFirstDisjoint, +} + +impl RangeOrdering { + fn cmp(a: &Range, b: &Range) -> Self { + use Ordering::*; + assert!(a.start <= a.end && b.start <= b.end); + match (a.start.cmp(&b.start), a.end.cmp(&b.end)) { + _ if a.end <= b.start => RangeOrdering::LeftFirstDisjoint, + _ if b.end <= a.start => RangeOrdering::RightFirstDisjoint, + (Less, Less) => RangeOrdering::LeftFirstOverlap, + (Equal, Less) | (Greater, Less) | (Greater, Equal) => RangeOrdering::LeftContained, + (Equal, Equal) => RangeOrdering::Equal, + (Equal, Greater) | (Less, Greater) | (Less, Equal) => RangeOrdering::RightContained, + (Greater, Greater) => RangeOrdering::RightFirstOverlap, + } + } +} + +#[derive(Debug, PartialEq, Eq)] +enum Joined { + One(ScanRange), + Two(ScanRange, ScanRange), + Three(ScanRange, ScanRange, ScanRange), +} + +fn join_nonoverlapping(left: ScanRange, right: ScanRange) -> Joined { + assert!(left.block_range().end <= right.block_range().start); + + if left.block_range().end == right.block_range().start { + if left.priority() == right.priority() { + Joined::One(ScanRange::from_parts( + left.block_range().start..right.block_range().end, + left.priority(), + )) + } else { + Joined::Two(left, right) + } + } else { + // there is a gap that will need to be filled + let gap = ScanRange::from_parts( + left.block_range().end..right.block_range().start, + ScanPriority::Historic, + ); + + match join_nonoverlapping(left, gap) { + Joined::One(merged) => join_nonoverlapping(merged, right), + Joined::Two(left, gap) => match join_nonoverlapping(gap, right) { + Joined::One(merged) => Joined::Two(left, merged), + Joined::Two(gap, right) => Joined::Three(left, gap, right), + _ => unreachable!(), + }, + _ => unreachable!(), + } + } +} + +fn insert(current: ScanRange, to_insert: ScanRange, force_rescans: bool) -> Joined { + fn join_overlapping(left: ScanRange, right: ScanRange, insert: Insert) -> Joined { + assert!( + left.block_range().start <= right.block_range().start + && left.block_range().end > right.block_range().start + ); + + // recompute the range dominance based upon the queue entry priorities + let dominance = match insert.on { + InsertOn::Left => dominance(&right.priority(), &left.priority(), insert), + InsertOn::Right => dominance(&left.priority(), &right.priority(), insert), + }; + + match dominance { + Dominance::Left => { + if let Some(right) = right.truncate_start(left.block_range().end) { + Joined::Two(left, right) + } else { + Joined::One(left) + } + } + Dominance::Equal => Joined::One(ScanRange::from_parts( + left.block_range().start..max(left.block_range().end, right.block_range().end), + left.priority(), + )), + Dominance::Right => match ( + left.truncate_end(right.block_range().start), + left.truncate_start(right.block_range().end), + ) { + (Some(before), Some(after)) => Joined::Three(before, right, after), + (Some(before), None) => Joined::Two(before, right), + (None, Some(after)) => Joined::Two(right, after), + (None, None) => Joined::One(right), + }, + } + } + + use RangeOrdering::*; + match RangeOrdering::cmp(to_insert.block_range(), current.block_range()) { + LeftFirstDisjoint => join_nonoverlapping(to_insert, current), + LeftFirstOverlap | RightContained => { + join_overlapping(to_insert, current, Insert::left(force_rescans)) + } + Equal => Joined::One(ScanRange::from_parts( + to_insert.block_range().clone(), + match dominance( + ¤t.priority(), + &to_insert.priority(), + Insert::right(force_rescans), + ) { + Dominance::Left | Dominance::Equal => current.priority(), + Dominance::Right => to_insert.priority(), + }, + )), + RightFirstOverlap | LeftContained => { + join_overlapping(current, to_insert, Insert::right(force_rescans)) + } + RightFirstDisjoint => join_nonoverlapping(current, to_insert), + } +} + +#[derive(Debug, Clone)] +#[cfg(feature = "unstable-spanning-tree")] +pub enum SpanningTree { + Leaf(ScanRange), + Parent { + span: Range, + left: Box, + right: Box, + }, +} + +#[cfg(feature = "unstable-spanning-tree")] +impl SpanningTree { + fn span(&self) -> Range { + match self { + SpanningTree::Leaf(entry) => entry.block_range().clone(), + SpanningTree::Parent { span, .. } => span.clone(), + } + } + + fn from_joined(joined: Joined) -> Self { + match joined { + Joined::One(entry) => SpanningTree::Leaf(entry), + Joined::Two(left, right) => SpanningTree::Parent { + span: left.block_range().start..right.block_range().end, + left: Box::new(SpanningTree::Leaf(left)), + right: Box::new(SpanningTree::Leaf(right)), + }, + Joined::Three(left, mid, right) => SpanningTree::Parent { + span: left.block_range().start..right.block_range().end, + left: Box::new(SpanningTree::Leaf(left)), + right: Box::new(SpanningTree::Parent { + span: mid.block_range().start..right.block_range().end, + left: Box::new(SpanningTree::Leaf(mid)), + right: Box::new(SpanningTree::Leaf(right)), + }), + }, + } + } + + fn from_insert( + left: Box, + right: Box, + to_insert: ScanRange, + insert: Insert, + ) -> Self { + let (left, right) = match insert.on { + InsertOn::Left => (Box::new(left.insert(to_insert, insert.force_rescan)), right), + InsertOn::Right => (left, Box::new(right.insert(to_insert, insert.force_rescan))), + }; + SpanningTree::Parent { + span: left.span().start..right.span().end, + left, + right, + } + } + + fn from_split( + left: Self, + right: Self, + to_insert: ScanRange, + split_point: BlockHeight, + force_rescans: bool, + ) -> Self { + let (l_insert, r_insert) = to_insert + .split_at(split_point) + .expect("Split point is within the range of to_insert"); + let left = Box::new(left.insert(l_insert, force_rescans)); + let right = Box::new(right.insert(r_insert, force_rescans)); + SpanningTree::Parent { + span: left.span().start..right.span().end, + left, + right, + } + } + + pub fn insert(self, to_insert: ScanRange, force_rescans: bool) -> Self { + match self { + SpanningTree::Leaf(cur) => Self::from_joined(insert(cur, to_insert, force_rescans)), + SpanningTree::Parent { span, left, right } => { + // This algorithm always preserves the existing partition point, and does not do + // any rebalancing or unification of ranges within the tree. This should be okay + // because `into_vec` performs such unification, and the tree being unbalanced + // should be fine given the relatively small number of ranges we should ordinarily + // be concerned with. + use RangeOrdering::*; + match RangeOrdering::cmp(&span, to_insert.block_range()) { + LeftFirstDisjoint => { + // extend the right-hand branch + Self::from_insert(left, right, to_insert, Insert::right(force_rescans)) + } + LeftFirstOverlap => { + let split_point = left.span().end; + if split_point > to_insert.block_range().start { + Self::from_split(*left, *right, to_insert, split_point, force_rescans) + } else { + // to_insert is fully contained in or equals the right child + Self::from_insert(left, right, to_insert, Insert::right(force_rescans)) + } + } + RightContained => { + // to_insert is fully contained within the current span, so we will insert + // into one or both sides + let split_point = left.span().end; + if to_insert.block_range().start >= split_point { + // to_insert is fully contained in the right + Self::from_insert(left, right, to_insert, Insert::right(force_rescans)) + } else if to_insert.block_range().end <= split_point { + // to_insert is fully contained in the left + Self::from_insert(left, right, to_insert, Insert::left(force_rescans)) + } else { + // to_insert must be split. + Self::from_split(*left, *right, to_insert, split_point, force_rescans) + } + } + Equal => { + let split_point = left.span().end; + if split_point > to_insert.block_range().start { + Self::from_split(*left, *right, to_insert, split_point, force_rescans) + } else { + // to_insert is fully contained in the right subtree + right.insert(to_insert, force_rescans) + } + } + LeftContained => { + // the current span is fully contained within to_insert, so we will extend + // or overwrite both sides + let split_point = left.span().end; + Self::from_split(*left, *right, to_insert, split_point, force_rescans) + } + RightFirstOverlap => { + let split_point = left.span().end; + if split_point < to_insert.block_range().end { + Self::from_split(*left, *right, to_insert, split_point, force_rescans) + } else { + // to_insert is fully contained in or equals the left child + Self::from_insert(left, right, to_insert, Insert::left(force_rescans)) + } + } + RightFirstDisjoint => { + // extend the left-hand branch + Self::from_insert(left, right, to_insert, Insert::left(force_rescans)) + } + } + } + } + } + + pub fn into_vec(self) -> Vec { + fn go(acc: &mut Vec, tree: SpanningTree) { + match tree { + SpanningTree::Leaf(entry) => { + if !entry.is_empty() { + if let Some(top) = acc.pop() { + match join_nonoverlapping(top, entry) { + Joined::One(merged) => acc.push(merged), + Joined::Two(l, r) => { + acc.push(l); + acc.push(r); + } + _ => unreachable!(), + } + } else { + acc.push(entry); + } + } + } + SpanningTree::Parent { left, right, .. } => { + go(acc, *left); + go(acc, *right); + } + } + } + + let mut acc = vec![]; + go(&mut acc, self); + acc + } +} + +#[cfg(any(test, feature = "test-dependencies"))] +pub mod testing { + use std::ops::Range; + + use zcash_primitives::consensus::BlockHeight; + + use crate::data_api::scanning::{ScanPriority, ScanRange}; + + pub fn scan_range(range: Range, priority: ScanPriority) -> ScanRange { + ScanRange::from_parts( + BlockHeight::from(range.start)..BlockHeight::from(range.end), + priority, + ) + } +} + +#[cfg(test)] +mod tests { + use std::ops::Range; + + use zcash_primitives::consensus::BlockHeight; + + use super::{join_nonoverlapping, testing::scan_range, Joined, RangeOrdering, SpanningTree}; + use crate::data_api::scanning::{ScanPriority, ScanRange}; + + #[test] + fn test_join_nonoverlapping() { + fn test_range(left: ScanRange, right: ScanRange, expected_joined: Joined) { + let joined = join_nonoverlapping(left, right); + + assert_eq!(joined, expected_joined); + } + + macro_rules! range { + ( $start:expr, $end:expr; $priority:ident ) => { + ScanRange::from_parts( + BlockHeight::from($start)..BlockHeight::from($end), + ScanPriority::$priority, + ) + }; + } + + macro_rules! joined { + ( + ($a_start:expr, $a_end:expr; $a_priority:ident) + ) => { + Joined::One( + range!($a_start, $a_end; $a_priority) + ) + }; + ( + ($a_start:expr, $a_end:expr; $a_priority:ident), + ($b_start:expr, $b_end:expr; $b_priority:ident) + ) => { + Joined::Two( + range!($a_start, $a_end; $a_priority), + range!($b_start, $b_end; $b_priority) + ) + }; + ( + ($a_start:expr, $a_end:expr; $a_priority:ident), + ($b_start:expr, $b_end:expr; $b_priority:ident), + ($c_start:expr, $c_end:expr; $c_priority:ident) + + ) => { + Joined::Three( + range!($a_start, $a_end; $a_priority), + range!($b_start, $b_end; $b_priority), + range!($c_start, $c_end; $c_priority) + ) + }; + } + + // Scan ranges have the same priority and + // line up. + test_range( + range!(1, 9; OpenAdjacent), + range!(9, 15; OpenAdjacent), + joined!( + (1, 15; OpenAdjacent) + ), + ); + + // Scan ranges have different priorities, + // so we cannot merge them even though they + // line up. + test_range( + range!(1, 9; OpenAdjacent), + range!(9, 15; ChainTip), + joined!( + (1, 9; OpenAdjacent), + (9, 15; ChainTip) + ), + ); + + // Scan ranges have the same priority but + // do not line up. + test_range( + range!(1, 9; OpenAdjacent), + range!(13, 15; OpenAdjacent), + joined!( + (1, 9; OpenAdjacent), + (9, 13; Historic), + (13, 15; OpenAdjacent) + ), + ); + + test_range( + range!(1, 9; Historic), + range!(13, 15; OpenAdjacent), + joined!( + (1, 13; Historic), + (13, 15; OpenAdjacent) + ), + ); + + test_range( + range!(1, 9; OpenAdjacent), + range!(13, 15; Historic), + joined!( + (1, 9; OpenAdjacent), + (9, 15; Historic) + ), + ); + } + + #[test] + fn range_ordering() { + use super::RangeOrdering::*; + // Equal + assert_eq!(RangeOrdering::cmp(&(0..1), &(0..1)), Equal); + + // Disjoint or contiguous + assert_eq!(RangeOrdering::cmp(&(0..1), &(1..2)), LeftFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(1..2), &(0..1)), RightFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(0..1), &(2..3)), LeftFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(2..3), &(0..1)), RightFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(1..2), &(2..2)), LeftFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(2..2), &(1..2)), RightFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(1..1), &(1..2)), LeftFirstDisjoint); + assert_eq!(RangeOrdering::cmp(&(1..2), &(1..1)), RightFirstDisjoint); + + // Contained + assert_eq!(RangeOrdering::cmp(&(1..2), &(0..3)), LeftContained); + assert_eq!(RangeOrdering::cmp(&(0..3), &(1..2)), RightContained); + assert_eq!(RangeOrdering::cmp(&(0..1), &(0..3)), LeftContained); + assert_eq!(RangeOrdering::cmp(&(0..3), &(0..1)), RightContained); + assert_eq!(RangeOrdering::cmp(&(2..3), &(0..3)), LeftContained); + assert_eq!(RangeOrdering::cmp(&(0..3), &(2..3)), RightContained); + + // Overlap + assert_eq!(RangeOrdering::cmp(&(0..2), &(1..3)), LeftFirstOverlap); + assert_eq!(RangeOrdering::cmp(&(1..3), &(0..2)), RightFirstOverlap); + } + + fn spanning_tree(to_insert: &[(Range, ScanPriority)]) -> Option { + to_insert.iter().fold(None, |acc, (range, priority)| { + let scan_range = scan_range(range.clone(), *priority); + match acc { + None => Some(SpanningTree::Leaf(scan_range)), + Some(t) => Some(t.insert(scan_range, false)), + } + }) + } + + #[test] + fn spanning_tree_insert_adjacent() { + use ScanPriority::*; + + let t = spanning_tree(&[ + (0..3, Historic), + (3..6, Scanned), + (6..8, ChainTip), + (8..10, ChainTip), + ]) + .unwrap(); + + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..3, Historic), + scan_range(3..6, Scanned), + scan_range(6..10, ChainTip), + ] + ); + } + + #[test] + fn spanning_tree_insert_overlaps() { + use ScanPriority::*; + + let t = spanning_tree(&[ + (0..3, Historic), + (2..5, Scanned), + (6..8, ChainTip), + (7..10, Scanned), + ]) + .unwrap(); + + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..2, Historic), + scan_range(2..5, Scanned), + scan_range(5..6, Historic), + scan_range(6..7, ChainTip), + scan_range(7..10, Scanned), + ] + ); + } + + #[test] + fn spanning_tree_insert_empty() { + use ScanPriority::*; + + let t = spanning_tree(&[ + (0..3, Historic), + (3..6, Scanned), + (6..6, FoundNote), + (6..8, Scanned), + (8..10, ChainTip), + ]) + .unwrap(); + + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..3, Historic), + scan_range(3..8, Scanned), + scan_range(8..10, ChainTip), + ] + ); + } + + #[test] + fn spanning_tree_insert_gaps() { + use ScanPriority::*; + + let t = spanning_tree(&[(0..3, Historic), (6..8, ChainTip)]).unwrap(); + + assert_eq!( + t.into_vec(), + vec![scan_range(0..6, Historic), scan_range(6..8, ChainTip),] + ); + + let t = spanning_tree(&[(0..3, Historic), (3..4, Verify), (6..8, ChainTip)]).unwrap(); + + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..3, Historic), + scan_range(3..4, Verify), + scan_range(4..6, Historic), + scan_range(6..8, ChainTip), + ] + ); + } + + #[test] + fn spanning_tree_insert_rfd_span() { + use ScanPriority::*; + + // This sequence of insertions causes a RightFirstDisjoint on the last insertion, + // which originally had a bug that caused the parent's span to only cover its left + // child. The bug was otherwise unobservable as the insertion logic was able to + // heal this specific kind of bug. + let t = spanning_tree(&[ + // 6..8 + (6..8, Scanned), + // 6..12 + // 6..8 8..12 + // 8..10 10..12 + (10..12, ChainTip), + // 3..12 + // 3..8 8..12 + // 3..6 6..8 8..10 10..12 + (3..6, Historic), + ]) + .unwrap(); + + assert_eq!(t.span(), (3.into())..(12.into())); + assert_eq!( + t.into_vec(), + vec![ + scan_range(3..6, Historic), + scan_range(6..8, Scanned), + scan_range(8..10, Historic), + scan_range(10..12, ChainTip), + ] + ); + } + + #[test] + fn spanning_tree_dominance() { + use ScanPriority::*; + + let t = spanning_tree(&[(0..3, Verify), (2..8, Scanned), (6..10, Verify)]).unwrap(); + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..2, Verify), + scan_range(2..6, Scanned), + scan_range(6..10, Verify), + ] + ); + + let t = spanning_tree(&[(0..3, Verify), (2..8, Historic), (6..10, Verify)]).unwrap(); + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..3, Verify), + scan_range(3..6, Historic), + scan_range(6..10, Verify), + ] + ); + + let t = spanning_tree(&[(0..3, Scanned), (2..8, Verify), (6..10, Scanned)]).unwrap(); + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..2, Scanned), + scan_range(2..6, Verify), + scan_range(6..10, Scanned), + ] + ); + + let t = spanning_tree(&[(0..3, Scanned), (2..8, Historic), (6..10, Scanned)]).unwrap(); + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..3, Scanned), + scan_range(3..6, Historic), + scan_range(6..10, Scanned), + ] + ); + + // a `ChainTip` insertion should not overwrite a scanned range. + let mut t = spanning_tree(&[(0..3, ChainTip), (3..5, Scanned), (5..7, ChainTip)]).unwrap(); + t = t.insert(scan_range(0..7, ChainTip), false); + assert_eq!( + t.into_vec(), + vec![ + scan_range(0..3, ChainTip), + scan_range(3..5, Scanned), + scan_range(5..7, ChainTip), + ] + ); + + let mut t = + spanning_tree(&[(280300..280310, FoundNote), (280310..280320, Scanned)]).unwrap(); + assert_eq!( + t.clone().into_vec(), + vec![ + scan_range(280300..280310, FoundNote), + scan_range(280310..280320, Scanned) + ] + ); + t = t.insert(scan_range(280300..280340, ChainTip), false); + assert_eq!( + t.into_vec(), + vec![ + scan_range(280300..280310, ChainTip), + scan_range(280310..280320, Scanned), + scan_range(280320..280340, ChainTip) + ] + ); + } + + #[test] + fn spanning_tree_insert_coalesce_scanned() { + use ScanPriority::*; + + let mut t = spanning_tree(&[ + (0..3, Historic), + (2..5, Scanned), + (6..8, ChainTip), + (7..10, Scanned), + ]) + .unwrap(); + + t = t.insert(scan_range(0..3, Scanned), false); + t = t.insert(scan_range(5..8, Scanned), false); + + assert_eq!(t.into_vec(), vec![scan_range(0..10, Scanned)]); + } + + #[test] + fn spanning_tree_force_rescans() { + use ScanPriority::*; + + let mut t = spanning_tree(&[ + (0..3, Historic), + (3..5, Scanned), + (5..7, ChainTip), + (7..10, Scanned), + ]) + .unwrap(); + + t = t.insert(scan_range(4..9, OpenAdjacent), true); + + let expected = vec![ + scan_range(0..3, Historic), + scan_range(3..4, Scanned), + scan_range(4..5, OpenAdjacent), + scan_range(5..7, ChainTip), + scan_range(7..9, OpenAdjacent), + scan_range(9..10, Scanned), + ]; + assert_eq!(t.clone().into_vec(), expected); + + // An insert of an ignored range should not override a scanned range; the existing + // priority should prevail, and so the expected state of the tree is unchanged. + t = t.insert(scan_range(2..5, Ignored), true); + assert_eq!(t.into_vec(), expected); + } +} diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index fba2c8f3fe..0acd4cbf3d 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -14,7 +14,7 @@ edition = "2021" rust-version = "1.65" [dependencies] -zcash_client_backend = { version = "=0.10.0-rc.2", path = "../zcash_client_backend", features = ["unstable-serialization"]} +zcash_client_backend = { version = "=0.10.0-rc.2", path = "../zcash_client_backend", features = ["unstable-serialization", "unstable-spanning-tree"]} zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false } diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index d1ba239cda..aa7050e699 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -1,16 +1,18 @@ use rusqlite::{self, named_params, types::Value, OptionalExtension}; use shardtree::error::ShardTreeError; -use std::cmp::{max, min, Ordering}; +use std::cmp::{max, min}; use std::collections::BTreeSet; -use std::ops::{Not, Range}; +use std::ops::Range; use std::rc::Rc; use tracing::{debug, trace}; use incrementalmerkletree::{Address, Position}; -use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; use zcash_primitives::consensus::{self, BlockHeight, NetworkUpgrade}; -use zcash_client_backend::data_api::SAPLING_SHARD_HEIGHT; +use zcash_client_backend::data_api::{ + scanning::{spanning_tree::SpanningTree, ScanPriority, ScanRange}, + SAPLING_SHARD_HEIGHT, +}; use crate::{ error::SqliteClientError, @@ -20,60 +22,16 @@ use crate::{ use super::wallet_birthday; -#[derive(Debug, Clone, Copy)] -enum InsertOn { - Left, - Right, -} - -struct Insert { - on: InsertOn, - force_rescan: bool, -} - -impl Insert { - fn left(force_rescan: bool) -> Self { - Insert { - on: InsertOn::Left, - force_rescan, - } - } - - fn right(force_rescan: bool) -> Self { - Insert { - on: InsertOn::Right, - force_rescan, - } - } -} - -impl Not for Insert { - type Output = Self; - - fn not(self) -> Self::Output { - Insert { - on: match self.on { - InsertOn::Left => InsertOn::Right, - InsertOn::Right => InsertOn::Left, - }, - force_rescan: self.force_rescan, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Dominance { - Left, - Right, - Equal, -} - -impl From for Dominance { - fn from(value: Insert) -> Self { - match value.on { - InsertOn::Left => Dominance::Left, - InsertOn::Right => Dominance::Right, - } +pub(crate) fn priority_code(priority: &ScanPriority) -> i64 { + use ScanPriority::*; + match priority { + Ignored => 0, + Scanned => 10, + Historic => 20, + OpenAdjacent => 30, + FoundNote => 40, + ChainTip => 50, + Verify => 60, } } @@ -91,19 +49,6 @@ pub(crate) fn parse_priority_code(code: i64) -> Option { } } -pub(crate) fn priority_code(priority: &ScanPriority) -> i64 { - use ScanPriority::*; - match priority { - Ignored => 0, - Scanned => 10, - Historic => 20, - OpenAdjacent => 30, - FoundNote => 40, - ChainTip => 50, - Verify => 60, - } -} - pub(crate) fn suggest_scan_ranges( conn: &rusqlite::Connection, min_priority: ScanPriority, @@ -135,335 +80,6 @@ pub(crate) fn suggest_scan_ranges( Ok(result) } -// This implements the dominance rule for range priority. If the inserted range's priority is -// `Verify`, this replaces any existing priority. Otherwise, if the current priority is -// `Scanned`, it remains as `Scanned`; and if the new priority is `Scanned`, it -// overrides any existing priority. -fn dominance(current: &ScanPriority, inserted: &ScanPriority, insert: Insert) -> Dominance { - match (current.cmp(inserted), (current, inserted)) { - (Ordering::Equal, _) => Dominance::Equal, - (_, (_, ScanPriority::Verify | ScanPriority::Scanned)) => Dominance::from(insert), - (_, (ScanPriority::Scanned, _)) if !insert.force_rescan => Dominance::from(!insert), - (Ordering::Less, _) => Dominance::from(insert), - (Ordering::Greater, _) => Dominance::from(!insert), - } -} - -/// In the comments for each alternative, `()` represents the left range and `[]` represents the right range. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum RangeOrdering { - /// `( ) [ ]` - LeftFirstDisjoint, - /// `( [ ) ]` - LeftFirstOverlap, - /// `[ ( ) ]` - LeftContained, - /// ```text - /// ( ) - /// [ ] - /// ``` - Equal, - /// `( [ ] )` - RightContained, - /// `[ ( ] )` - RightFirstOverlap, - /// `[ ] ( )` - RightFirstDisjoint, -} - -impl RangeOrdering { - fn cmp(a: &Range, b: &Range) -> Self { - use Ordering::*; - assert!(a.start <= a.end && b.start <= b.end); - match (a.start.cmp(&b.start), a.end.cmp(&b.end)) { - _ if a.end <= b.start => RangeOrdering::LeftFirstDisjoint, - _ if b.end <= a.start => RangeOrdering::RightFirstDisjoint, - (Less, Less) => RangeOrdering::LeftFirstOverlap, - (Equal, Less) | (Greater, Less) | (Greater, Equal) => RangeOrdering::LeftContained, - (Equal, Equal) => RangeOrdering::Equal, - (Equal, Greater) | (Less, Greater) | (Less, Equal) => RangeOrdering::RightContained, - (Greater, Greater) => RangeOrdering::RightFirstOverlap, - } - } -} - -#[derive(Debug, PartialEq, Eq)] -enum Joined { - One(ScanRange), - Two(ScanRange, ScanRange), - Three(ScanRange, ScanRange, ScanRange), -} - -fn join_nonoverlapping(left: ScanRange, right: ScanRange) -> Joined { - assert!(left.block_range().end <= right.block_range().start); - - if left.block_range().end == right.block_range().start { - if left.priority() == right.priority() { - Joined::One(ScanRange::from_parts( - left.block_range().start..right.block_range().end, - left.priority(), - )) - } else { - Joined::Two(left, right) - } - } else { - // there is a gap that will need to be filled - let gap = ScanRange::from_parts( - left.block_range().end..right.block_range().start, - ScanPriority::Historic, - ); - - match join_nonoverlapping(left, gap) { - Joined::One(merged) => join_nonoverlapping(merged, right), - Joined::Two(left, gap) => match join_nonoverlapping(gap, right) { - Joined::One(merged) => Joined::Two(left, merged), - Joined::Two(gap, right) => Joined::Three(left, gap, right), - _ => unreachable!(), - }, - _ => unreachable!(), - } - } -} - -fn insert(current: ScanRange, to_insert: ScanRange, force_rescans: bool) -> Joined { - fn join_overlapping(left: ScanRange, right: ScanRange, insert: Insert) -> Joined { - assert!( - left.block_range().start <= right.block_range().start - && left.block_range().end > right.block_range().start - ); - - // recompute the range dominance based upon the queue entry priorities - let dominance = match insert.on { - InsertOn::Left => dominance(&right.priority(), &left.priority(), insert), - InsertOn::Right => dominance(&left.priority(), &right.priority(), insert), - }; - - match dominance { - Dominance::Left => { - if let Some(right) = right.truncate_start(left.block_range().end) { - Joined::Two(left, right) - } else { - Joined::One(left) - } - } - Dominance::Equal => Joined::One(ScanRange::from_parts( - left.block_range().start..max(left.block_range().end, right.block_range().end), - left.priority(), - )), - Dominance::Right => match ( - left.truncate_end(right.block_range().start), - left.truncate_start(right.block_range().end), - ) { - (Some(before), Some(after)) => Joined::Three(before, right, after), - (Some(before), None) => Joined::Two(before, right), - (None, Some(after)) => Joined::Two(right, after), - (None, None) => Joined::One(right), - }, - } - } - - use RangeOrdering::*; - match RangeOrdering::cmp(to_insert.block_range(), current.block_range()) { - LeftFirstDisjoint => join_nonoverlapping(to_insert, current), - LeftFirstOverlap | RightContained => { - join_overlapping(to_insert, current, Insert::left(force_rescans)) - } - Equal => Joined::One(ScanRange::from_parts( - to_insert.block_range().clone(), - match dominance( - ¤t.priority(), - &to_insert.priority(), - Insert::right(force_rescans), - ) { - Dominance::Left | Dominance::Equal => current.priority(), - Dominance::Right => to_insert.priority(), - }, - )), - RightFirstOverlap | LeftContained => { - join_overlapping(current, to_insert, Insert::right(force_rescans)) - } - RightFirstDisjoint => join_nonoverlapping(current, to_insert), - } -} - -#[derive(Debug, Clone)] -enum SpanningTree { - Leaf(ScanRange), - Parent { - span: Range, - left: Box, - right: Box, - }, -} - -impl SpanningTree { - fn span(&self) -> Range { - match self { - SpanningTree::Leaf(entry) => entry.block_range().clone(), - SpanningTree::Parent { span, .. } => span.clone(), - } - } - - fn from_joined(joined: Joined) -> Self { - match joined { - Joined::One(entry) => SpanningTree::Leaf(entry), - Joined::Two(left, right) => SpanningTree::Parent { - span: left.block_range().start..right.block_range().end, - left: Box::new(SpanningTree::Leaf(left)), - right: Box::new(SpanningTree::Leaf(right)), - }, - Joined::Three(left, mid, right) => SpanningTree::Parent { - span: left.block_range().start..right.block_range().end, - left: Box::new(SpanningTree::Leaf(left)), - right: Box::new(SpanningTree::Parent { - span: mid.block_range().start..right.block_range().end, - left: Box::new(SpanningTree::Leaf(mid)), - right: Box::new(SpanningTree::Leaf(right)), - }), - }, - } - } - - fn from_insert( - left: Box, - right: Box, - to_insert: ScanRange, - insert: Insert, - ) -> Self { - let (left, right) = match insert.on { - InsertOn::Left => (Box::new(left.insert(to_insert, insert.force_rescan)), right), - InsertOn::Right => (left, Box::new(right.insert(to_insert, insert.force_rescan))), - }; - SpanningTree::Parent { - span: left.span().start..right.span().end, - left, - right, - } - } - - fn from_split( - left: Self, - right: Self, - to_insert: ScanRange, - split_point: BlockHeight, - force_rescans: bool, - ) -> Self { - let (l_insert, r_insert) = to_insert - .split_at(split_point) - .expect("Split point is within the range of to_insert"); - let left = Box::new(left.insert(l_insert, force_rescans)); - let right = Box::new(right.insert(r_insert, force_rescans)); - SpanningTree::Parent { - span: left.span().start..right.span().end, - left, - right, - } - } - - fn insert(self, to_insert: ScanRange, force_rescans: bool) -> Self { - match self { - SpanningTree::Leaf(cur) => Self::from_joined(insert(cur, to_insert, force_rescans)), - SpanningTree::Parent { span, left, right } => { - // This algorithm always preserves the existing partition point, and does not do - // any rebalancing or unification of ranges within the tree. This should be okay - // because `into_vec` performs such unification, and the tree being unbalanced - // should be fine given the relatively small number of ranges we should ordinarily - // be concerned with. - use RangeOrdering::*; - match RangeOrdering::cmp(&span, to_insert.block_range()) { - LeftFirstDisjoint => { - // extend the right-hand branch - Self::from_insert(left, right, to_insert, Insert::right(force_rescans)) - } - LeftFirstOverlap => { - let split_point = left.span().end; - if split_point > to_insert.block_range().start { - Self::from_split(*left, *right, to_insert, split_point, force_rescans) - } else { - // to_insert is fully contained in or equals the right child - Self::from_insert(left, right, to_insert, Insert::right(force_rescans)) - } - } - RightContained => { - // to_insert is fully contained within the current span, so we will insert - // into one or both sides - let split_point = left.span().end; - if to_insert.block_range().start >= split_point { - // to_insert is fully contained in the right - Self::from_insert(left, right, to_insert, Insert::right(force_rescans)) - } else if to_insert.block_range().end <= split_point { - // to_insert is fully contained in the left - Self::from_insert(left, right, to_insert, Insert::left(force_rescans)) - } else { - // to_insert must be split. - Self::from_split(*left, *right, to_insert, split_point, force_rescans) - } - } - Equal => { - let split_point = left.span().end; - if split_point > to_insert.block_range().start { - Self::from_split(*left, *right, to_insert, split_point, force_rescans) - } else { - // to_insert is fully contained in the right subtree - right.insert(to_insert, force_rescans) - } - } - LeftContained => { - // the current span is fully contained within to_insert, so we will extend - // or overwrite both sides - let split_point = left.span().end; - Self::from_split(*left, *right, to_insert, split_point, force_rescans) - } - RightFirstOverlap => { - let split_point = left.span().end; - if split_point < to_insert.block_range().end { - Self::from_split(*left, *right, to_insert, split_point, force_rescans) - } else { - // to_insert is fully contained in or equals the left child - Self::from_insert(left, right, to_insert, Insert::left(force_rescans)) - } - } - RightFirstDisjoint => { - // extend the left-hand branch - Self::from_insert(left, right, to_insert, Insert::left(force_rescans)) - } - } - } - } - } - - fn into_vec(self) -> Vec { - fn go(acc: &mut Vec, tree: SpanningTree) { - match tree { - SpanningTree::Leaf(entry) => { - if !entry.is_empty() { - if let Some(top) = acc.pop() { - match join_nonoverlapping(top, entry) { - Joined::One(merged) => acc.push(merged), - Joined::Two(l, r) => { - acc.push(l); - acc.push(r); - } - _ => unreachable!(), - } - } else { - acc.push(entry); - } - } - } - SpanningTree::Parent { left, right, .. } => { - go(acc, *left); - go(acc, *right); - } - } - } - - let mut acc = vec![]; - go(&mut acc, self); - acc - } -} - pub(crate) fn insert_queue_entries<'a>( conn: &rusqlite::Connection, entries: impl Iterator, @@ -883,14 +499,12 @@ pub(crate) fn update_chain_tip( #[cfg(test)] pub(crate) mod tests { - use std::ops::Range; - use incrementalmerkletree::{frontier::Frontier, Hashable, Level, Position}; use secrecy::SecretVec; use zcash_client_backend::data_api::{ chain::CommitmentTreeRoot, - scanning::{ScanPriority, ScanRange}, + scanning::{spanning_tree::testing::scan_range, ScanPriority}, AccountBirthday, Ratio, WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }; @@ -909,406 +523,6 @@ pub(crate) mod tests { VERIFY_LOOKAHEAD, }; - use super::{join_nonoverlapping, Joined, RangeOrdering, SpanningTree}; - - #[test] - fn test_join_nonoverlapping() { - fn test_range(left: ScanRange, right: ScanRange, expected_joined: Joined) { - let joined = join_nonoverlapping(left, right); - - assert_eq!(joined, expected_joined); - } - - macro_rules! range { - ( $start:expr, $end:expr; $priority:ident ) => { - ScanRange::from_parts( - BlockHeight::from($start)..BlockHeight::from($end), - ScanPriority::$priority, - ) - }; - } - - macro_rules! joined { - ( - ($a_start:expr, $a_end:expr; $a_priority:ident) - ) => { - Joined::One( - range!($a_start, $a_end; $a_priority) - ) - }; - ( - ($a_start:expr, $a_end:expr; $a_priority:ident), - ($b_start:expr, $b_end:expr; $b_priority:ident) - ) => { - Joined::Two( - range!($a_start, $a_end; $a_priority), - range!($b_start, $b_end; $b_priority) - ) - }; - ( - ($a_start:expr, $a_end:expr; $a_priority:ident), - ($b_start:expr, $b_end:expr; $b_priority:ident), - ($c_start:expr, $c_end:expr; $c_priority:ident) - - ) => { - Joined::Three( - range!($a_start, $a_end; $a_priority), - range!($b_start, $b_end; $b_priority), - range!($c_start, $c_end; $c_priority) - ) - }; - } - - // Scan ranges have the same priority and - // line up. - test_range( - range!(1, 9; OpenAdjacent), - range!(9, 15; OpenAdjacent), - joined!( - (1, 15; OpenAdjacent) - ), - ); - - // Scan ranges have different priorities, - // so we cannot merge them even though they - // line up. - test_range( - range!(1, 9; OpenAdjacent), - range!(9, 15; ChainTip), - joined!( - (1, 9; OpenAdjacent), - (9, 15; ChainTip) - ), - ); - - // Scan ranges have the same priority but - // do not line up. - test_range( - range!(1, 9; OpenAdjacent), - range!(13, 15; OpenAdjacent), - joined!( - (1, 9; OpenAdjacent), - (9, 13; Historic), - (13, 15; OpenAdjacent) - ), - ); - - test_range( - range!(1, 9; Historic), - range!(13, 15; OpenAdjacent), - joined!( - (1, 13; Historic), - (13, 15; OpenAdjacent) - ), - ); - - test_range( - range!(1, 9; OpenAdjacent), - range!(13, 15; Historic), - joined!( - (1, 9; OpenAdjacent), - (9, 15; Historic) - ), - ); - } - - #[test] - fn range_ordering() { - use super::RangeOrdering::*; - // Equal - assert_eq!(RangeOrdering::cmp(&(0..1), &(0..1)), Equal); - - // Disjoint or contiguous - assert_eq!(RangeOrdering::cmp(&(0..1), &(1..2)), LeftFirstDisjoint); - assert_eq!(RangeOrdering::cmp(&(1..2), &(0..1)), RightFirstDisjoint); - assert_eq!(RangeOrdering::cmp(&(0..1), &(2..3)), LeftFirstDisjoint); - assert_eq!(RangeOrdering::cmp(&(2..3), &(0..1)), RightFirstDisjoint); - assert_eq!(RangeOrdering::cmp(&(1..2), &(2..2)), LeftFirstDisjoint); - assert_eq!(RangeOrdering::cmp(&(2..2), &(1..2)), RightFirstDisjoint); - assert_eq!(RangeOrdering::cmp(&(1..1), &(1..2)), LeftFirstDisjoint); - assert_eq!(RangeOrdering::cmp(&(1..2), &(1..1)), RightFirstDisjoint); - - // Contained - assert_eq!(RangeOrdering::cmp(&(1..2), &(0..3)), LeftContained); - assert_eq!(RangeOrdering::cmp(&(0..3), &(1..2)), RightContained); - assert_eq!(RangeOrdering::cmp(&(0..1), &(0..3)), LeftContained); - assert_eq!(RangeOrdering::cmp(&(0..3), &(0..1)), RightContained); - assert_eq!(RangeOrdering::cmp(&(2..3), &(0..3)), LeftContained); - assert_eq!(RangeOrdering::cmp(&(0..3), &(2..3)), RightContained); - - // Overlap - assert_eq!(RangeOrdering::cmp(&(0..2), &(1..3)), LeftFirstOverlap); - assert_eq!(RangeOrdering::cmp(&(1..3), &(0..2)), RightFirstOverlap); - } - - fn scan_range(range: Range, priority: ScanPriority) -> ScanRange { - ScanRange::from_parts( - BlockHeight::from(range.start)..BlockHeight::from(range.end), - priority, - ) - } - - fn spanning_tree(to_insert: &[(Range, ScanPriority)]) -> Option { - to_insert.iter().fold(None, |acc, (range, priority)| { - let scan_range = scan_range(range.clone(), *priority); - match acc { - None => Some(SpanningTree::Leaf(scan_range)), - Some(t) => Some(t.insert(scan_range, false)), - } - }) - } - - #[test] - fn spanning_tree_insert_adjacent() { - use ScanPriority::*; - - let t = spanning_tree(&[ - (0..3, Historic), - (3..6, Scanned), - (6..8, ChainTip), - (8..10, ChainTip), - ]) - .unwrap(); - - assert_eq!( - t.into_vec(), - vec![ - scan_range(0..3, Historic), - scan_range(3..6, Scanned), - scan_range(6..10, ChainTip), - ] - ); - } - - #[test] - fn spanning_tree_insert_overlaps() { - use ScanPriority::*; - - let t = spanning_tree(&[ - (0..3, Historic), - (2..5, Scanned), - (6..8, ChainTip), - (7..10, Scanned), - ]) - .unwrap(); - - assert_eq!( - t.into_vec(), - vec![ - scan_range(0..2, Historic), - scan_range(2..5, Scanned), - scan_range(5..6, Historic), - scan_range(6..7, ChainTip), - scan_range(7..10, Scanned), - ] - ); - } - - #[test] - fn spanning_tree_insert_empty() { - use ScanPriority::*; - - let t = spanning_tree(&[ - (0..3, Historic), - (3..6, Scanned), - (6..6, FoundNote), - (6..8, Scanned), - (8..10, ChainTip), - ]) - .unwrap(); - - assert_eq!( - t.into_vec(), - vec![ - scan_range(0..3, Historic), - scan_range(3..8, Scanned), - scan_range(8..10, ChainTip), - ] - ); - } - - #[test] - fn spanning_tree_insert_gaps() { - use ScanPriority::*; - - let t = spanning_tree(&[(0..3, Historic), (6..8, ChainTip)]).unwrap(); - - assert_eq!( - t.into_vec(), - vec![scan_range(0..6, Historic), scan_range(6..8, ChainTip),] - ); - - let t = spanning_tree(&[(0..3, Historic), (3..4, Verify), (6..8, ChainTip)]).unwrap(); - - assert_eq!( - t.into_vec(), - vec![ - scan_range(0..3, Historic), - scan_range(3..4, Verify), - scan_range(4..6, Historic), - scan_range(6..8, ChainTip), - ] - ); - } - - #[test] - fn spanning_tree_insert_rfd_span() { - use ScanPriority::*; - - // This sequence of insertions causes a RightFirstDisjoint on the last insertion, - // which originally had a bug that caused the parent's span to only cover its left - // child. The bug was otherwise unobservable as the insertion logic was able to - // heal this specific kind of bug. - let t = spanning_tree(&[ - // 6..8 - (6..8, Scanned), - // 6..12 - // 6..8 8..12 - // 8..10 10..12 - (10..12, ChainTip), - // 3..12 - // 3..8 8..12 - // 3..6 6..8 8..10 10..12 - (3..6, Historic), - ]) - .unwrap(); - - assert_eq!(t.span(), (3.into())..(12.into())); - assert_eq!( - t.into_vec(), - vec![ - scan_range(3..6, Historic), - scan_range(6..8, Scanned), - scan_range(8..10, Historic), - scan_range(10..12, ChainTip), - ] - ); - } - - #[test] - fn spanning_tree_dominance() { - use ScanPriority::*; - - let t = spanning_tree(&[(0..3, Verify), (2..8, Scanned), (6..10, Verify)]).unwrap(); - assert_eq!( - t.into_vec(), - vec![ - scan_range(0..2, Verify), - scan_range(2..6, Scanned), - scan_range(6..10, Verify), - ] - ); - - let t = spanning_tree(&[(0..3, Verify), (2..8, Historic), (6..10, Verify)]).unwrap(); - assert_eq!( - t.into_vec(), - vec![ - scan_range(0..3, Verify), - scan_range(3..6, Historic), - scan_range(6..10, Verify), - ] - ); - - let t = spanning_tree(&[(0..3, Scanned), (2..8, Verify), (6..10, Scanned)]).unwrap(); - assert_eq!( - t.into_vec(), - vec![ - scan_range(0..2, Scanned), - scan_range(2..6, Verify), - scan_range(6..10, Scanned), - ] - ); - - let t = spanning_tree(&[(0..3, Scanned), (2..8, Historic), (6..10, Scanned)]).unwrap(); - assert_eq!( - t.into_vec(), - vec![ - scan_range(0..3, Scanned), - scan_range(3..6, Historic), - scan_range(6..10, Scanned), - ] - ); - - // a `ChainTip` insertion should not overwrite a scanned range. - let mut t = spanning_tree(&[(0..3, ChainTip), (3..5, Scanned), (5..7, ChainTip)]).unwrap(); - t = t.insert(scan_range(0..7, ChainTip), false); - assert_eq!( - t.into_vec(), - vec![ - scan_range(0..3, ChainTip), - scan_range(3..5, Scanned), - scan_range(5..7, ChainTip), - ] - ); - - let mut t = - spanning_tree(&[(280300..280310, FoundNote), (280310..280320, Scanned)]).unwrap(); - assert_eq!( - t.clone().into_vec(), - vec![ - scan_range(280300..280310, FoundNote), - scan_range(280310..280320, Scanned) - ] - ); - t = t.insert(scan_range(280300..280340, ChainTip), false); - assert_eq!( - t.into_vec(), - vec![ - scan_range(280300..280310, ChainTip), - scan_range(280310..280320, Scanned), - scan_range(280320..280340, ChainTip) - ] - ); - } - - #[test] - fn spanning_tree_insert_coalesce_scanned() { - use ScanPriority::*; - - let mut t = spanning_tree(&[ - (0..3, Historic), - (2..5, Scanned), - (6..8, ChainTip), - (7..10, Scanned), - ]) - .unwrap(); - - t = t.insert(scan_range(0..3, Scanned), false); - t = t.insert(scan_range(5..8, Scanned), false); - - assert_eq!(t.into_vec(), vec![scan_range(0..10, Scanned)]); - } - - #[test] - fn spanning_tree_force_rescans() { - use ScanPriority::*; - - let mut t = spanning_tree(&[ - (0..3, Historic), - (3..5, Scanned), - (5..7, ChainTip), - (7..10, Scanned), - ]) - .unwrap(); - - t = t.insert(scan_range(4..9, OpenAdjacent), true); - - let expected = vec![ - scan_range(0..3, Historic), - scan_range(3..4, Scanned), - scan_range(4..5, OpenAdjacent), - scan_range(5..7, ChainTip), - scan_range(7..9, OpenAdjacent), - scan_range(9..10, Scanned), - ]; - assert_eq!(t.clone().into_vec(), expected); - - // An insert of an ignored range should not override a scanned range; the existing - // priority should prevail, and so the expected state of the tree is unchanged. - t = t.insert(scan_range(2..5, Ignored), true); - assert_eq!(t.into_vec(), expected); - } - #[test] fn scan_complete() { use ScanPriority::*; From 6ebd66da765665bc1ea59351add51ed7d597249d Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 18 Sep 2023 13:43:26 -0600 Subject: [PATCH 0182/1122] zcash_client_sqlite: Remove inner join between `transactions` and `utxos` from `v_transactions` Under normal usage conditions, the `transactions` table is not currently populated for transactions involving transparent UTXOs, and so this join was always resulting in transparent UTXO information being filtered out from the transaction history. Fixes [zcash/ZcashLightClientKit#1271] --- zcash_client_sqlite/CHANGELOG.md | 3 + zcash_client_sqlite/src/wallet/init.rs | 55 ++--- .../src/wallet/init/migrations.rs | 15 +- .../v_transactions_transparent_history.rs | 190 ++++++++++++++++++ 4 files changed, 234 insertions(+), 29 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/v_transactions_transparent_history.rs diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 2e1c9533d4..4a9f71d064 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -42,6 +42,9 @@ and this library adheres to Rust's notion of - `FsBlockDbError::CacheMiss` - `zcash_client_sqlite::FsBlockDb::write_block_metadata` now overwrites any existing metadata entries that have the same height as a new entry. +- The `v_transactions` and `v_tx_outputs` views now return the 32-byte + transaction identifier for transactions instead of the internal database + id. ### Removed - The empty `wallet::transact` module has been removed. diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 18c27f303e..ce132ba48e 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -436,8 +436,9 @@ mod tests { WITH notes AS ( SELECT sapling_received_notes.account AS account_id, - sapling_received_notes.tx AS id_tx, - 2 AS pool, + transactions.block AS block, + transactions.txid AS txid, + 2 AS pool, sapling_received_notes.value AS value, CASE WHEN sapling_received_notes.is_change THEN 1 @@ -452,32 +453,36 @@ mod tests { THEN 0 ELSE 1 END AS memo_present - FROM sapling_received_notes + FROM sapling_received_notes + JOIN transactions + ON transactions.id_tx = sapling_received_notes.tx UNION SELECT utxos.received_by_account AS account_id, - transactions.id_tx AS id_tx, + utxos.height AS block, + utxos.prevout_txid AS txid, 0 AS pool, utxos.value_zat AS value, 0 AS is_change, 1 AS received_count, 0 AS memo_present FROM utxos - JOIN transactions - ON transactions.txid = utxos.prevout_txid UNION SELECT sapling_received_notes.account AS account_id, - sapling_received_notes.spent AS id_tx, - 2 AS pool, + transactions.block AS block, + transactions.txid AS txid, + 2 AS pool, -sapling_received_notes.value AS value, 0 AS is_change, 0 AS received_count, 0 AS memo_present - FROM sapling_received_notes + FROM sapling_received_notes + JOIN transactions + ON transactions.id_tx = sapling_received_notes.spent WHERE sapling_received_notes.spent IS NOT NULL ), sent_note_counts AS ( SELECT sent_notes.from_account AS account_id, - sent_notes.tx AS id_tx, + transactions.txid AS txid, COUNT(DISTINCT sent_notes.id_note) as sent_notes, SUM( CASE @@ -487,21 +492,22 @@ mod tests { END ) AS memo_count FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx LEFT JOIN sapling_received_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = (sapling_received_notes.tx, 2, sapling_received_notes.output_index) WHERE sapling_received_notes.is_change IS NULL OR sapling_received_notes.is_change = 0 - GROUP BY account_id, id_tx + GROUP BY account_id, txid ), blocks_max_height AS ( SELECT MAX(blocks.height) as max_height FROM blocks ) SELECT notes.account_id AS account_id, - transactions.id_tx AS id_tx, - transactions.block AS mined_height, + notes.block AS mined_height, + notes.txid AS txid, transactions.tx_index AS tx_index, - transactions.txid AS txid, transactions.expiry_height AS expiry_height, transactions.raw AS raw, SUM(notes.value) AS account_balance_delta, @@ -515,17 +521,18 @@ mod tests { blocks.height IS NULL AND transactions.expiry_height <= blocks_max_height.max_height ) AS expired_unmined - FROM transactions - JOIN notes ON notes.id_tx = transactions.id_tx + FROM notes + LEFT OUTER JOIN transactions + ON notes.txid = transactions.txid JOIN blocks_max_height LEFT JOIN blocks ON blocks.height = transactions.block LEFT JOIN sent_note_counts ON sent_note_counts.account_id = notes.account_id - AND sent_note_counts.id_tx = notes.id_tx - GROUP BY notes.account_id, transactions.id_tx".to_owned(), + AND sent_note_counts.txid = notes.txid + GROUP BY notes.account_id, notes.txid".to_owned(), // v_tx_outputs "CREATE VIEW v_tx_outputs AS - SELECT sapling_received_notes.tx AS id_tx, + SELECT transactions.txid AS txid, 2 AS output_pool, sapling_received_notes.output_index AS output_index, sent_notes.from_account AS from_account, @@ -535,11 +542,13 @@ mod tests { sapling_received_notes.is_change AS is_change, sapling_received_notes.memo AS memo FROM sapling_received_notes + JOIN transactions + ON transactions.id_tx = sapling_received_notes.tx LEFT JOIN sent_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = (sapling_received_notes.tx, 2, sent_notes.output_index) UNION - SELECT transactions.id_tx AS id_tx, + SELECT utxos.prevout_txid AS txid, 0 AS output_pool, utxos.prevout_idx AS output_index, NULL AS from_account, @@ -549,10 +558,8 @@ mod tests { false AS is_change, NULL AS memo FROM utxos - JOIN transactions - ON transactions.txid = utxos.prevout_txid UNION - SELECT sent_notes.tx AS id_tx, + SELECT transactions.txid AS txid, sent_notes.output_pool AS output_pool, sent_notes.output_index AS output_index, sent_notes.from_account AS from_account, @@ -562,6 +569,8 @@ mod tests { false AS is_change, sent_notes.memo AS memo FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx LEFT JOIN sapling_received_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = (sapling_received_notes.tx, 2, sapling_received_notes.output_index) diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 03381ba2cf..7fa3fa0aa9 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -12,6 +12,7 @@ mod ufvk_support; mod utxos_table; mod v_sapling_shard_unscanned_ranges; mod v_transactions_net; +mod v_transactions_transparent_history; mod wallet_summaries; use schemer_rusqlite::RusqliteMigration; @@ -39,12 +40,13 @@ pub(super) fn all_migrations( // received_notes_nullable_nf // / | \ // shardtree_support nullifier_map sapling_memo_consistency - // | - // add_account_birthdays - // | - // v_sapling_shard_unscanned_ranges - // | - // wallet_summaries + // | | + // add_account_birthdays | + // | | + // v_sapling_shard_unscanned_ranges | + // | | + // wallet_summaries | + // v_transactions_transparent_history vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -76,5 +78,6 @@ pub(super) fn all_migrations( params: params.clone(), }), Box::new(wallet_summaries::Migration), + Box::new(v_transactions_transparent_history::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_transparent_history.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_transparent_history.rs new file mode 100644 index 0000000000..28f6c149f3 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_transparent_history.rs @@ -0,0 +1,190 @@ +//! This migration reworks transaction history views to correctly include history +//! of transparent utxos for which we lack complete transaction information. + +use std::collections::HashSet; + +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::WalletMigrationError; + +use super::sapling_memo_consistency; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xaa0a4168_b41b_44c5_a47d_c4c66603cfab); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [sapling_memo_consistency::MIGRATION_ID] + .into_iter() + .collect() + } + + fn description(&self) -> &'static str { + "Updates transaction history views to fix potential errors in transparent history." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + transaction.execute_batch( + "DROP VIEW v_transactions; + CREATE VIEW v_transactions AS + WITH + notes AS ( + SELECT sapling_received_notes.account AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 2 AS pool, + sapling_received_notes.value AS value, + CASE + WHEN sapling_received_notes.is_change THEN 1 + ELSE 0 + END AS is_change, + CASE + WHEN sapling_received_notes.is_change THEN 0 + ELSE 1 + END AS received_count, + CASE + WHEN (sapling_received_notes.memo IS NULL OR sapling_received_notes.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present + FROM sapling_received_notes + JOIN transactions + ON transactions.id_tx = sapling_received_notes.tx + UNION + SELECT utxos.received_by_account AS account_id, + utxos.height AS block, + utxos.prevout_txid AS txid, + 0 AS pool, + utxos.value_zat AS value, + 0 AS is_change, + 1 AS received_count, + 0 AS memo_present + FROM utxos + UNION + SELECT sapling_received_notes.account AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 2 AS pool, + -sapling_received_notes.value AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM sapling_received_notes + JOIN transactions + ON transactions.id_tx = sapling_received_notes.spent + WHERE sapling_received_notes.spent IS NOT NULL + ), + sent_note_counts AS ( + SELECT sent_notes.from_account AS account_id, + transactions.txid AS txid, + COUNT(DISTINCT sent_notes.id_note) as sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6') + THEN 0 + ELSE 1 + END + ) AS memo_count + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN sapling_received_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + WHERE sapling_received_notes.is_change IS NULL + OR sapling_received_notes.is_change = 0 + GROUP BY account_id, txid + ), + blocks_max_height AS ( + SELECT MAX(blocks.height) as max_height FROM blocks + ) + SELECT notes.account_id AS account_id, + notes.block AS mined_height, + notes.txid AS txid, + transactions.tx_index AS tx_index, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.is_change) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( + blocks.height IS NULL + AND transactions.expiry_height <= blocks_max_height.max_height + ) AS expired_unmined + FROM notes + LEFT OUTER JOIN transactions + ON notes.txid = transactions.txid + JOIN blocks_max_height + LEFT JOIN blocks ON blocks.height = transactions.block + LEFT JOIN sent_note_counts + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.txid = notes.txid + GROUP BY notes.account_id, notes.txid; + + DROP VIEW v_tx_outputs; + CREATE VIEW v_tx_outputs AS + SELECT transactions.txid AS txid, + 2 AS output_pool, + sapling_received_notes.output_index AS output_index, + sent_notes.from_account AS from_account, + sapling_received_notes.account AS to_account, + NULL AS to_address, + sapling_received_notes.value AS value, + sapling_received_notes.is_change AS is_change, + sapling_received_notes.memo AS memo + FROM sapling_received_notes + JOIN transactions + ON transactions.id_tx = sapling_received_notes.tx + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sent_notes.output_index) + UNION + SELECT utxos.prevout_txid AS txid, + 0 AS output_pool, + utxos.prevout_idx AS output_index, + NULL AS from_account, + utxos.received_by_account AS to_account, + utxos.address AS to_address, + utxos.value_zat AS value, + false AS is_change, + NULL AS memo + FROM utxos + UNION + SELECT transactions.txid AS txid, + sent_notes.output_pool AS output_pool, + sent_notes.output_index AS output_index, + sent_notes.from_account AS from_account, + sapling_received_notes.account AS to_account, + sent_notes.to_address AS to_address, + sent_notes.value AS value, + false AS is_change, + sent_notes.memo AS memo + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN sapling_received_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + WHERE sapling_received_notes.is_change IS NULL + OR sapling_received_notes.is_change = 0;" + ).map_err(WalletMigrationError::from) + } + + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + panic!("This migration cannot be reverted."); + } +} From 15bfb41773c8be7dce1fe3cb91cfc2297c9892a9 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 18 Sep 2023 14:13:05 -0600 Subject: [PATCH 0183/1122] Apply suggestions from code review. Co-Authored-By: str4d Co-Authored-By: Daira Hopwood --- zcash_client_sqlite/CHANGELOG.md | 9 ++++++--- zcash_client_sqlite/src/wallet/init.rs | 7 ++----- .../migrations/v_transactions_transparent_history.rs | 7 ++----- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 4a9f71d064..41d874b49f 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -42,9 +42,10 @@ and this library adheres to Rust's notion of - `FsBlockDbError::CacheMiss` - `zcash_client_sqlite::FsBlockDb::write_block_metadata` now overwrites any existing metadata entries that have the same height as a new entry. -- The `v_transactions` and `v_tx_outputs` views now return the 32-byte - transaction identifier for transactions instead of the internal database - id. +- The `v_transactions` and `v_tx_outputs` views no longer return the + internal database identifier for the transaction. The `txid` column + should be used instead. The `tx_index`, `expiry_height`, `raw` and + `fee_paid` columns may now be null for received transparent transactions. ### Removed - The empty `wallet::transact` module has been removed. @@ -70,6 +71,8 @@ and this library adheres to Rust's notion of - `WalletDb::get_transaction` no longer returns an error when called on a transaction that has not yet been mined, unless the transaction's consensus branch ID cannot be determined by other means. +- Fixed an error in `v_transactions` wherein received transparent outputs did not + result in a transaction entry appearing in the transaction history. ## [0.7.1] - 2023-05-17 diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index ce132ba48e..7d3544c0d9 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -478,7 +478,6 @@ mod tests { FROM sapling_received_notes JOIN transactions ON transactions.id_tx = sapling_received_notes.spent - WHERE sapling_received_notes.spent IS NOT NULL ), sent_note_counts AS ( SELECT sent_notes.from_account AS account_id, @@ -497,8 +496,7 @@ mod tests { LEFT JOIN sapling_received_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = (sapling_received_notes.tx, 2, sapling_received_notes.output_index) - WHERE sapling_received_notes.is_change IS NULL - OR sapling_received_notes.is_change = 0 + WHERE COALESCE(sapling_received_notes.is_change, 0) = 0 GROUP BY account_id, txid ), blocks_max_height AS ( @@ -574,8 +572,7 @@ mod tests { LEFT JOIN sapling_received_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = (sapling_received_notes.tx, 2, sapling_received_notes.output_index) - WHERE sapling_received_notes.is_change IS NULL - OR sapling_received_notes.is_change = 0".to_owned(), + WHERE COALESCE(sapling_received_notes.is_change, 0) = 0".to_owned(), ]; let mut views_query = st diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_transparent_history.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_transparent_history.rs index 28f6c149f3..4354a03d03 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_transparent_history.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_transparent_history.rs @@ -82,7 +82,6 @@ impl RusqliteMigration for Migration { FROM sapling_received_notes JOIN transactions ON transactions.id_tx = sapling_received_notes.spent - WHERE sapling_received_notes.spent IS NOT NULL ), sent_note_counts AS ( SELECT sent_notes.from_account AS account_id, @@ -101,8 +100,7 @@ impl RusqliteMigration for Migration { LEFT JOIN sapling_received_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = (sapling_received_notes.tx, 2, sapling_received_notes.output_index) - WHERE sapling_received_notes.is_change IS NULL - OR sapling_received_notes.is_change = 0 + WHERE COALESCE(sapling_received_notes.is_change, 0) = 0 GROUP BY account_id, txid ), blocks_max_height AS ( @@ -179,8 +177,7 @@ impl RusqliteMigration for Migration { LEFT JOIN sapling_received_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = (sapling_received_notes.tx, 2, sapling_received_notes.output_index) - WHERE sapling_received_notes.is_change IS NULL - OR sapling_received_notes.is_change = 0;" + WHERE COALESCE(sapling_received_notes.is_change, 0) = 0;" ).map_err(WalletMigrationError::from) } From c0e29ad103ef774b9aa3af68b80cdb164ed649e1 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 19 Sep 2023 09:20:44 -0600 Subject: [PATCH 0184/1122] zcash_client_sqlite: fix join to blocks table in `v_transactions` This allows us to display additional block information when a transparent UTXO was provided for a block that we have scanned. Also, this ensures that memos from a wallet to itself are not double-counted. Co-authored-by: str4d --- zcash_client_sqlite/src/wallet/init.rs | 8 ++++---- .../v_transactions_transparent_history.rs | 20 +++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 7d3544c0d9..5ea7a7625d 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -485,7 +485,7 @@ mod tests { COUNT(DISTINCT sent_notes.id_note) as sent_notes, SUM( CASE - WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6') + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR sapling_received_notes.tx IS NOT NULL) THEN 0 ELSE 1 END @@ -517,13 +517,13 @@ mod tests { blocks.time AS block_time, ( blocks.height IS NULL - AND transactions.expiry_height <= blocks_max_height.max_height + AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height ) AS expired_unmined FROM notes - LEFT OUTER JOIN transactions + LEFT JOIN transactions ON notes.txid = transactions.txid JOIN blocks_max_height - LEFT JOIN blocks ON blocks.height = transactions.block + LEFT JOIN blocks ON blocks.height = notes.block LEFT JOIN sent_note_counts ON sent_note_counts.account_id = notes.account_id AND sent_note_counts.txid = notes.txid diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_transparent_history.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_transparent_history.rs index 4354a03d03..477d58e292 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_transparent_history.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_transparent_history.rs @@ -89,7 +89,7 @@ impl RusqliteMigration for Migration { COUNT(DISTINCT sent_notes.id_note) as sent_notes, SUM( CASE - WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6') + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR sapling_received_notes.tx IS NOT NULL) THEN 0 ELSE 1 END @@ -121,19 +121,21 @@ impl RusqliteMigration for Migration { blocks.time AS block_time, ( blocks.height IS NULL - AND transactions.expiry_height <= blocks_max_height.max_height + AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height ) AS expired_unmined FROM notes - LEFT OUTER JOIN transactions + LEFT JOIN transactions ON notes.txid = transactions.txid JOIN blocks_max_height - LEFT JOIN blocks ON blocks.height = transactions.block + LEFT JOIN blocks ON blocks.height = notes.block LEFT JOIN sent_note_counts ON sent_note_counts.account_id = notes.account_id AND sent_note_counts.txid = notes.txid - GROUP BY notes.account_id, notes.txid; + GROUP BY notes.account_id, notes.txid;" + )?; - DROP VIEW v_tx_outputs; + transaction.execute_batch( + "DROP VIEW v_tx_outputs; CREATE VIEW v_tx_outputs AS SELECT transactions.txid AS txid, 2 AS output_pool, @@ -177,8 +179,10 @@ impl RusqliteMigration for Migration { LEFT JOIN sapling_received_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = (sapling_received_notes.tx, 2, sapling_received_notes.output_index) - WHERE COALESCE(sapling_received_notes.is_change, 0) = 0;" - ).map_err(WalletMigrationError::from) + WHERE COALESCE(sapling_received_notes.is_change, 0) = 0;", + )?; + + Ok(()) } fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { From 6d2c53111aee8a5a3419563e8a08bcefbadce359 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 19 Sep 2023 14:56:01 -0600 Subject: [PATCH 0185/1122] zcash_client_backend 0.10.0-rc.3 --- Cargo.lock | 2 +- zcash_client_backend/CHANGELOG.md | 2 +- zcash_client_backend/Cargo.toml | 2 +- zcash_client_sqlite/Cargo.toml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4547bfefad..80e4b8f7ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2946,7 +2946,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" -version = "0.10.0-rc.2" +version = "0.10.0-rc.3" dependencies = [ "assert_matches", "base64", diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index a4f86265a1..2280d6f859 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -7,7 +7,7 @@ and this library adheres to Rust's notion of ## [Unreleased] -## [0.10.0-rc.1] - 2023-09-08 +## [0.10.0-rc.3] - 2023-09-19 ### Notable Changes - `zcash_client_backend` now supports out-of-order scanning of blockchain history. diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index c38f851975..5e90c52655 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_backend" description = "APIs for creating shielded Zcash light clients" -version = "0.10.0-rc.2" +version = "0.10.0-rc.3" authors = [ "Jack Grigg ", "Kris Nuttycombe " diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 0acd4cbf3d..03ac42b985 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -14,7 +14,7 @@ edition = "2021" rust-version = "1.65" [dependencies] -zcash_client_backend = { version = "=0.10.0-rc.2", path = "../zcash_client_backend", features = ["unstable-serialization", "unstable-spanning-tree"]} +zcash_client_backend = { version = "=0.10.0-rc.3", path = "../zcash_client_backend", features = ["unstable-serialization", "unstable-spanning-tree"]} zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false } @@ -62,7 +62,7 @@ tempfile = "3.5.0" zcash_note_encryption = "0.4" zcash_proofs = { version = "=0.13.0-rc.1", path = "../zcash_proofs" } zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", features = ["test-dependencies"] } -zcash_client_backend = { version = "=0.10.0-rc.2", path = "../zcash_client_backend", features = ["test-dependencies", "unstable-serialization"] } +zcash_client_backend = { version = "=0.10.0-rc.3", path = "../zcash_client_backend", features = ["test-dependencies", "unstable-serialization", "unstable-spanning-tree"] } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } [features] From e7b3e9c04e3a35fed775275653d4cdc3c029b112 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 19 Sep 2023 15:01:52 -0600 Subject: [PATCH 0186/1122] zcash_client_sqlite 0.8.0-rc.4 --- Cargo.lock | 2 +- zcash_client_sqlite/CHANGELOG.md | 16 ++++++++++++---- zcash_client_sqlite/Cargo.toml | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 80e4b8f7ed..d2392353e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2988,7 +2988,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" -version = "0.8.0-rc.3" +version = "0.8.0-rc.4" dependencies = [ "assert_matches", "bs58", diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 41d874b49f..957de54a66 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -7,7 +7,13 @@ and this library adheres to Rust's notion of ## [Unreleased] -## [0.8.0-rc.3] - 2023-09-12 +## [0.8.0-rc.4] - 2023-09-19 + +### Notable Changes +- The `v_transactions` and `v_tx_outputs` views have changed in terms of what + columns are returned, and which result columns may be null. Please see the + `Changed` section below for additional details. + ### Added - `zcash_client_sqlite::commitment_tree` Types related to management of note commitment trees using the `shardtree` crate. @@ -43,9 +49,11 @@ and this library adheres to Rust's notion of - `zcash_client_sqlite::FsBlockDb::write_block_metadata` now overwrites any existing metadata entries that have the same height as a new entry. - The `v_transactions` and `v_tx_outputs` views no longer return the - internal database identifier for the transaction. The `txid` column - should be used instead. The `tx_index`, `expiry_height`, `raw` and - `fee_paid` columns may now be null for received transparent transactions. + internal database identifier for the transaction. The `txid` column should + be used instead. The `tx_index`, `expiry_height`, `raw`, `fee_paid`, and + `expired_unmined` columns will be null for received transparent + transactions, in addition to the other columns that were previously + permitted to be null. ### Removed - The empty `wallet::transact` module has been removed. diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 03ac42b985..16368a8543 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_sqlite" description = "An SQLite-based Zcash light client" -version = "0.8.0-rc.3" +version = "0.8.0-rc.4" authors = [ "Jack Grigg ", "Kris Nuttycombe " From b69d80f25d583ae9084b29c1a94ee451954997ab Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 20 Sep 2023 09:58:58 -0600 Subject: [PATCH 0187/1122] zcash_client_sqlite: Remove use of boolean constants from `v_tx_outputs` At present, [zcash/zcash-android-wallet-sdk] supports Android API 27, which bundles SQLite 3.19. SQLite support for the `TRUE` and `FALSE` constants were introduced in SQLite 3.23, so we cannot currently use these constants and retain support for Android API 27. This version support limitation applies only to the `v_transactions` and `v_tx_outputs` views, which are considered part of the public API of this crate; other use of more recent SQLite features is fine because they rely upon the SQLite bundled via our use of the `rusqlite` crate and feature compatibility is verified via the unit tests of this crate. --- zcash_client_sqlite/src/wallet/init.rs | 4 +- .../src/wallet/init/migrations.rs | 4 + .../v_tx_outputs_use_legacy_false.rs | 92 +++++++++++++++++++ 3 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/v_tx_outputs_use_legacy_false.rs diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 5ea7a7625d..e8c4341511 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -553,7 +553,7 @@ mod tests { utxos.received_by_account AS to_account, utxos.address AS to_address, utxos.value_zat AS value, - false AS is_change, + 0 AS is_change, NULL AS memo FROM utxos UNION @@ -564,7 +564,7 @@ mod tests { sapling_received_notes.account AS to_account, sent_notes.to_address AS to_address, sent_notes.value AS value, - false AS is_change, + 0 AS is_change, sent_notes.memo AS memo FROM sent_notes JOIN transactions diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 7fa3fa0aa9..0deaba2995 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -13,6 +13,7 @@ mod utxos_table; mod v_sapling_shard_unscanned_ranges; mod v_transactions_net; mod v_transactions_transparent_history; +mod v_tx_outputs_use_legacy_false; mod wallet_summaries; use schemer_rusqlite::RusqliteMigration; @@ -47,6 +48,8 @@ pub(super) fn all_migrations( // | | // wallet_summaries | // v_transactions_transparent_history + // | + // v_tx_outputs_use_legacy_false vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -79,5 +82,6 @@ pub(super) fn all_migrations( }), Box::new(wallet_summaries::Migration), Box::new(v_transactions_transparent_history::Migration), + Box::new(v_tx_outputs_use_legacy_false::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_tx_outputs_use_legacy_false.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_tx_outputs_use_legacy_false.rs new file mode 100644 index 0000000000..d4788871bd --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_tx_outputs_use_legacy_false.rs @@ -0,0 +1,92 @@ +//! This migration revises the `v_tx_outputs` view to support SQLite 3.19.x +//! which did not define `TRUE` and `FALSE` constants. This is required in +//! order to support Android API 27 + +use std::collections::HashSet; + +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::WalletMigrationError; + +use super::v_transactions_transparent_history; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xb3e21434_286f_41f3_8d71_44cce968ab2b); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [v_transactions_transparent_history::MIGRATION_ID] + .into_iter() + .collect() + } + + fn description(&self) -> &'static str { + "Updates v_tx_outputs to remove use of `true` and `false` constants for legacy SQLite version support." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + transaction.execute_batch( + "DROP VIEW v_tx_outputs; + CREATE VIEW v_tx_outputs AS + SELECT transactions.txid AS txid, + 2 AS output_pool, + sapling_received_notes.output_index AS output_index, + sent_notes.from_account AS from_account, + sapling_received_notes.account AS to_account, + NULL AS to_address, + sapling_received_notes.value AS value, + sapling_received_notes.is_change AS is_change, + sapling_received_notes.memo AS memo + FROM sapling_received_notes + JOIN transactions + ON transactions.id_tx = sapling_received_notes.tx + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sent_notes.output_index) + UNION + SELECT utxos.prevout_txid AS txid, + 0 AS output_pool, + utxos.prevout_idx AS output_index, + NULL AS from_account, + utxos.received_by_account AS to_account, + utxos.address AS to_address, + utxos.value_zat AS value, + 0 AS is_change, + NULL AS memo + FROM utxos + UNION + SELECT transactions.txid AS txid, + sent_notes.output_pool AS output_pool, + sent_notes.output_index AS output_index, + sent_notes.from_account AS from_account, + sapling_received_notes.account AS to_account, + sent_notes.to_address AS to_address, + sent_notes.value AS value, + 0 AS is_change, + sent_notes.memo AS memo + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN sapling_received_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + WHERE COALESCE(sapling_received_notes.is_change, 0) = 0;", + )?; + + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + panic!("This migration cannot be reverted."); + } +} From 996fea70ac57bec2b90f555aca0c068532fe2940 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 20 Sep 2023 12:39:36 -0600 Subject: [PATCH 0188/1122] zcash_client_backend: Add detail to documentation of `suggest_scan_ranges` Fixes #971 --- zcash_client_backend/src/data_api.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 86cf4c4e33..e50fac38cc 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -257,7 +257,11 @@ pub trait WalletRead { /// tree size information for each block; or else the scan is likely to fail if notes belonging /// to the wallet are detected. /// - /// The returned range(s) may include block heights beyond the current chain tip. + /// The returned range(s) may include block heights beyond the current chain tip. Ranges are + /// returned in order of descending priority, and higher-priority ranges should always be + /// scanned before lower-priority ranges; in particular, ranges with [`ScanPriority::Verify`] + /// priority must always be scanned first in order to avoid blockchain continuity errors in the + /// case of a reorg. /// /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock fn suggest_scan_ranges(&self) -> Result, Self::Error>; From d5dc4c6d9cdfc5382bc2d4112b121b9cd6564c09 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 20 Sep 2023 12:53:45 -0600 Subject: [PATCH 0189/1122] zcash_client_backend: Document `data_api::ScannedBlock` Fixes #885 --- zcash_client_backend/src/data_api.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index e50fac38cc..d7431680df 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -456,6 +456,7 @@ pub struct ScannedBlock { } impl ScannedBlock { + /// Constructs a new `ScannedBlock` pub fn from_parts( metadata: BlockMetadata, block_time: u32, @@ -472,34 +473,53 @@ impl ScannedBlock { } } + /// Returns the height of the block that was scanned. pub fn height(&self) -> BlockHeight { self.metadata.block_height } + /// Returns the block hash of the block that was scanned. pub fn block_hash(&self) -> BlockHash { self.metadata.block_hash } + /// Returns the block time of the block that was scanned. pub fn block_time(&self) -> u32 { self.block_time } + /// Returns the metadata describing the state of the note commitment trees as of the end of the + /// scanned block. + /// + /// The metadata returned from this method is guaranteed to be consistent with what is returned + /// by [`Self::height`] and [`Self::block_hash`]. pub fn metadata(&self) -> &BlockMetadata { &self.metadata } + /// Returns the list of transactions from the block that are relevant to the wallet. pub fn transactions(&self) -> &[WalletTx] { &self.transactions } + /// Returns the vector of Sapling nullifiers for each transaction in the block. + /// + /// The returned tuple is keyed by both transaction ID and the index of the transaction within + /// the block, so that either the txid or the combination of the block hash available from + /// [`Self::block_hash`] and returned transaction index may be used to uniquely identify the + /// transaction, depending upon the needs of the caller. pub fn sapling_nullifier_map(&self) -> &[(TxId, u16, Vec)] { &self.sapling_nullifier_map } + /// Returns the ordered list of Sapling note commitments to be added to the note commitment + /// tree. pub fn sapling_commitments(&self) -> &[(sapling::Node, Retention)] { &self.sapling_commitments } + /// Consumes `self` and returns the list of Sapling note commitments associated with the + /// scanned block as an owned value. pub fn into_sapling_commitments(self) -> Vec<(sapling::Node, Retention)> { self.sapling_commitments } From bdc6886e2dec60bb55b9d6f69a6ad485ac408f02 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 20 Sep 2023 13:21:53 -0600 Subject: [PATCH 0190/1122] Apply suggestions from code review. Co-authored-by: Daira Emma Hopwood --- zcash_client_backend/src/data_api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index d7431680df..e324e42917 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -483,7 +483,7 @@ impl ScannedBlock { self.metadata.block_hash } - /// Returns the block time of the block that was scanned. + /// Returns the block time of the block that was scanned, as a Unix timestamp in seconds. pub fn block_time(&self) -> u32 { self.block_time } From 2d8ece3003de114dfa5bbc2c083ea5242da16b4e Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 20 Sep 2023 13:24:53 -0600 Subject: [PATCH 0191/1122] Fix broken intra-doc link --- zcash_client_backend/src/data_api.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index e324e42917..18646c44d1 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -264,6 +264,7 @@ pub trait WalletRead { /// case of a reorg. /// /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock + /// [`ScanPriority::Verify`]: crate::data_api::scanning::ScanPriority fn suggest_scan_ranges(&self) -> Result, Self::Error>; /// Returns the default target height (for the block in which a new From 9e4fa96dd7b927476640a0f2bfe96c136e793fb8 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 21 Sep 2023 22:24:46 +0000 Subject: [PATCH 0192/1122] zcash_client_sqlite: Clean up existing transparent input test --- zcash_client_sqlite/src/wallet.rs | 84 +++++++++++++++---------------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index bcc1d4f9f2..c1872ab477 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1918,7 +1918,6 @@ mod tests { #[cfg(feature = "transparent-inputs")] use { - secrecy::Secret, zcash_client_backend::{ data_api::WalletWrite, encoding::AddressCodec, wallet::WalletTransparentOutput, }, @@ -1963,12 +1962,11 @@ mod tests { fn put_received_transparent_utxo() { use crate::testing::TestBuilder; - let mut st = TestBuilder::new().build(); + let mut st = TestBuilder::new() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); - // Add an account to the wallet - let seed = Secret::new([0u8; 32].to_vec()); - let birthday = AccountBirthday::from_sapling_activation(&st.network()); - let (account_id, _usk) = st.wallet_mut().create_account(&seed, birthday).unwrap(); + let (account_id, _, _) = st.test_account().unwrap(); let uaddr = st .wallet() .get_current_address(account_id) @@ -1976,63 +1974,63 @@ mod tests { .unwrap(); let taddr = uaddr.transparent().unwrap(); + let height_1 = BlockHeight::from_u32(12345); let bal_absent = st .wallet() - .get_transparent_balances(account_id, BlockHeight::from_u32(12345)) + .get_transparent_balances(account_id, height_1) .unwrap(); assert!(bal_absent.is_empty()); - let utxo = WalletTransparentOutput::from_parts( - OutPoint::new([1u8; 32], 1), - TxOut { - value: Amount::from_u64(100000).unwrap(), - script_pubkey: taddr.script(), - }, - BlockHeight::from_u32(12345), - ) - .unwrap(); + // Create a fake transparent output. + let value = Amount::from_u64(100000).unwrap(); + let outpoint = OutPoint::new([1u8; 32], 1); + let txout = TxOut { + value, + script_pubkey: taddr.script(), + }; + // Pretend the output's transaction was mined at `height_1`. + let utxo = + WalletTransparentOutput::from_parts(outpoint.clone(), txout.clone(), height_1).unwrap(); let res0 = st.wallet_mut().put_received_transparent_utxo(&utxo); assert_matches!(res0, Ok(_)); + // Confirm that we see the output unspent as of `height_1`. + assert_matches!( + st.wallet().get_unspent_transparent_outputs( + taddr, + height_1, + &[] + ).as_deref(), + Ok(&[ref ret]) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo.outpoint(), utxo.txout(), height_1) + ); + // Change the mined height of the UTXO and upsert; we should get back - // the same utxoid - let utxo2 = WalletTransparentOutput::from_parts( - OutPoint::new([1u8; 32], 1), - TxOut { - value: Amount::from_u64(100000).unwrap(), - script_pubkey: taddr.script(), - }, - BlockHeight::from_u32(34567), - ) - .unwrap(); + // the same `UtxoId`. + let height_2 = BlockHeight::from_u32(34567); + let utxo2 = WalletTransparentOutput::from_parts(outpoint, txout, height_2).unwrap(); let res1 = st.wallet_mut().put_received_transparent_utxo(&utxo2); assert_matches!(res1, Ok(id) if id == res0.unwrap()); + // Confirm that we no longer see any unspent outputs as of `height_1`. assert_matches!( - st.wallet().get_unspent_transparent_outputs( - taddr, - BlockHeight::from_u32(12345), - &[] - ), - Ok(utxos) if utxos.is_empty() + st.wallet() + .get_unspent_transparent_outputs(taddr, height_1, &[]) + .as_deref(), + Ok(&[]) ); + // If we include `height_2` then the output is returned. assert_matches!( - st.wallet().get_unspent_transparent_outputs( - taddr, - BlockHeight::from_u32(34567), - &[] - ), - Ok(utxos) if { - utxos.len() == 1 && - utxos.iter().any(|rutxo| rutxo.height() == utxo2.height()) - } + st.wallet() + .get_unspent_transparent_outputs(taddr, height_2, &[]) + .as_deref(), + Ok(&[ref ret]) if (ret.outpoint(), ret.txout(), ret.height()) == (utxo.outpoint(), utxo.txout(), height_2) ); assert_matches!( - st.wallet().get_transparent_balances(account_id, BlockHeight::from_u32(34567)), - Ok(h) if h.get(taddr) == Amount::from_u64(100000).ok().as_ref() + st.wallet().get_transparent_balances(account_id, height_2), + Ok(h) if h.get(taddr) == Some(&value) ); // Artificially delete the address from the addresses table so that From 71e38fe190857ddcc742f517b787b7763bb4fd30 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 22 Sep 2023 00:09:00 +0000 Subject: [PATCH 0193/1122] zcash_client_sqlite: Enable `TestState` to mine wallet transactions --- zcash_client_backend/src/proto.rs | 27 +++++++-- zcash_client_sqlite/src/testing.rs | 70 +++++++++++++++++------ zcash_client_sqlite/src/wallet/sapling.rs | 12 ++-- 3 files changed, 80 insertions(+), 29 deletions(-) diff --git a/zcash_client_backend/src/proto.rs b/zcash_client_backend/src/proto.rs index ee21a8e9fa..a938174046 100644 --- a/zcash_client_backend/src/proto.rs +++ b/zcash_client_backend/src/proto.rs @@ -116,10 +116,8 @@ impl compact_formats::CompactSaplingOutput { } } -impl From> - for compact_formats::CompactSaplingOutput -{ - fn from(out: sapling::OutputDescription) -> compact_formats::CompactSaplingOutput { +impl From<&sapling::OutputDescription> for compact_formats::CompactSaplingOutput { + fn from(out: &sapling::OutputDescription) -> compact_formats::CompactSaplingOutput { compact_formats::CompactSaplingOutput { cmu: out.cmu().to_bytes().to_vec(), ephemeral_key: out.ephemeral_key().as_ref().to_vec(), @@ -146,6 +144,27 @@ impl compact_formats::CompactSaplingSpend { } } +impl From<&sapling::SpendDescription> + for compact_formats::CompactSaplingSpend +{ + fn from(spend: &sapling::SpendDescription) -> compact_formats::CompactSaplingSpend { + compact_formats::CompactSaplingSpend { + nf: spend.nullifier().to_vec(), + } + } +} + +impl From<&orchard::Action> for compact_formats::CompactOrchardAction { + fn from(action: &orchard::Action) -> compact_formats::CompactOrchardAction { + compact_formats::CompactOrchardAction { + nullifier: action.nullifier().to_bytes().to_vec(), + cmx: action.cmx().to_bytes().to_vec(), + ephemeral_key: action.encrypted_note().epk_bytes.to_vec(), + ciphertext: action.encrypted_note().enc_ciphertext[..COMPACT_NOTE_SIZE].to_vec(), + } + } +} + impl service::TreeState { /// Deserializes and returns the Sapling note commitment tree field of the tree state. pub fn sapling_tree(&self) -> io::Result> { diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index a9c5cffe0e..5bc160ba09 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -14,7 +14,7 @@ use tempfile::NamedTempFile; #[cfg(feature = "unstable")] use tempfile::TempDir; -use zcash_client_backend::data_api::AccountBalance; +use zcash_client_backend::data_api::{AccountBalance, WalletRead}; #[allow(deprecated)] use zcash_client_backend::{ address::RecipientAddress, @@ -35,7 +35,7 @@ use zcash_client_backend::{ wallet::OvkPolicy, zip321, }; -use zcash_note_encryption::{Domain, COMPACT_NOTE_SIZE}; +use zcash_note_encryption::Domain; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight, Network, NetworkUpgrade, Parameters}, @@ -270,14 +270,34 @@ where (height, res) } + /// Creates a fake block at the expected next height containing only the wallet + /// transaction with the given txid, and inserts it into the cache. + /// + /// This generated block will be treated as the latest block, and subsequent calls to + /// [`Self::generate_next_block`] (or similar) will build on it. + pub(crate) fn generate_next_block_including( + &mut self, + txid: TxId, + ) -> (BlockHeight, Cache::InsertResult) { + let tx = self + .wallet() + .get_transaction(txid) + .expect("TxId should exist in the wallet"); + + // Index 0 is by definition a coinbase transaction, and the wallet doesn't + // construct coinbase transactions. So we pretend here that the block has a + // coinbase transaction that does not have shielded coinbase outputs. + self.generate_next_block_from_tx(1, &tx) + } + /// Creates a fake block at the expected next height containing only the given /// transaction, and inserts it into the cache. - /// This assumes that the transaction only has Sapling spends and outputs. /// /// This generated block will be treated as the latest block, and subsequent calls to /// [`Self::generate_next_block`] will build on it. pub(crate) fn generate_next_block_from_tx( &mut self, + tx_index: usize, tx: &Transaction, ) -> (BlockHeight, Cache::InsertResult) { let (height, prev_hash, initial_sapling_tree_size) = self @@ -285,7 +305,14 @@ where .map(|(prev_height, prev_hash, end_size)| (prev_height + 1, prev_hash, end_size)) .unwrap_or_else(|| (self.sapling_activation_height(), BlockHash([0; 32]), 0)); - let cb = fake_compact_block_from_tx(height, prev_hash, tx, initial_sapling_tree_size); + let cb = fake_compact_block_from_tx( + height, + prev_hash, + tx_index, + tx, + initial_sapling_tree_size, + 0, + ); let res = self.cache.insert(&cb); self.latest_cached_block = Some(( @@ -728,35 +755,42 @@ pub(crate) fn fake_compact_block( } /// Create a fake CompactBlock at the given height containing only the given transaction. -/// This assumes that the transaction only has Sapling spends and outputs. pub(crate) fn fake_compact_block_from_tx( height: BlockHeight, prev_hash: BlockHash, + tx_index: usize, tx: &Transaction, initial_sapling_tree_size: u32, + initial_orchard_tree_size: u32, ) -> CompactBlock { - // Create a fake CompactTx + // Create a fake CompactTx containing the transaction. let mut ctx = CompactTx { + index: tx_index as u64, hash: tx.txid().as_ref().to_vec(), ..Default::default() }; if let Some(bundle) = tx.sapling_bundle() { for spend in bundle.shielded_spends() { - ctx.spends.push(CompactSaplingSpend { - nf: spend.nullifier().to_vec(), - }); + ctx.spends.push(spend.into()); } for output in bundle.shielded_outputs() { - ctx.outputs.push(CompactSaplingOutput { - cmu: output.cmu().to_bytes().to_vec(), - ephemeral_key: output.ephemeral_key().0.to_vec(), - ciphertext: output.enc_ciphertext()[..COMPACT_NOTE_SIZE].to_vec(), - }); + ctx.outputs.push(output.into()); + } + } + if let Some(bundle) = tx.orchard_bundle() { + for action in bundle.actions() { + ctx.actions.push(action.into()); } } - fake_compact_block_from_compact_tx(ctx, height, prev_hash, initial_sapling_tree_size) + fake_compact_block_from_compact_tx( + ctx, + height, + prev_hash, + initial_sapling_tree_size, + initial_orchard_tree_size, + ) } /// Create a fake CompactBlock at the given height, spending a single note from the @@ -833,7 +867,7 @@ pub(crate) fn fake_compact_block_spending( } }); - fake_compact_block_from_compact_tx(ctx, height, prev_hash, initial_sapling_tree_size) + fake_compact_block_from_compact_tx(ctx, height, prev_hash, initial_sapling_tree_size, 0) } pub(crate) fn fake_compact_block_from_compact_tx( @@ -841,6 +875,7 @@ pub(crate) fn fake_compact_block_from_compact_tx( height: BlockHeight, prev_hash: BlockHash, initial_sapling_tree_size: u32, + initial_orchard_tree_size: u32, ) -> CompactBlock { let mut rng = OsRng; let mut cb = CompactBlock { @@ -857,7 +892,8 @@ pub(crate) fn fake_compact_block_from_compact_tx( cb.chain_metadata = Some(compact::ChainMetadata { sapling_commitment_tree_size: initial_sapling_tree_size + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), - ..Default::default() + orchard_commitment_tree_size: initial_orchard_tree_size + + cb.vtx.iter().map(|tx| tx.actions.len() as u32).sum::(), }); cb } diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index e5ccb7d655..6bf84cf007 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -821,9 +821,8 @@ pub(crate) mod tests { NonZeroU32::new(10).unwrap(), ) .unwrap(); - let tx = &st.wallet().get_transaction(txid).unwrap(); - let (h, _) = st.generate_next_block_from_tx(tx); + let (h, _) = st.generate_next_block_including(txid); st.scan_cached_blocks(h, 1); // TODO: send to an account so that we can check its balance. @@ -938,9 +937,8 @@ pub(crate) mod tests { NonZeroU32::new(1).unwrap(), ) .unwrap(); - let tx2 = &st.wallet().get_transaction(txid2).unwrap(); - let (h, _) = st.generate_next_block_from_tx(tx2); + let (h, _) = st.generate_next_block_including(txid2); st.scan_cached_blocks(h, 1); // TODO: send to an account so that we can check its balance. @@ -1200,7 +1198,6 @@ pub(crate) mod tests { NonZeroU32::new(1).unwrap(), ) .unwrap(); - let tx = &st.wallet().get_transaction(txid).unwrap(); let amount_left = (value - (amount_sent + fee_rule.fixed_fee().try_into().unwrap()).unwrap()).unwrap(); @@ -1211,7 +1208,7 @@ pub(crate) mod tests { // We spent the only note so we only have pending change. assert_eq!(st.get_total_balance(AccountId::from(0)), pending_change); - let (h, _) = st.generate_next_block_from_tx(tx); + let (h, _) = st.generate_next_block_including(txid); st.scan_cached_blocks(h, 1); assert_eq!(st.get_total_balance(AccountId::from(1)), amount_sent); @@ -1330,9 +1327,8 @@ pub(crate) mod tests { NonZeroU32::new(1).unwrap(), ) .unwrap(); - let tx = &st.wallet().get_transaction(txid).unwrap(); - let (h, _) = st.generate_next_block_from_tx(tx); + let (h, _) = st.generate_next_block_including(txid); st.scan_cached_blocks(h, 1); // TODO: send to an account so that we can check its balance. From cd6c9627195e98c40f001f15c3b312c67554d8f6 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 22 Sep 2023 01:00:34 +0000 Subject: [PATCH 0194/1122] zcash_client_sqlite: Write a test for transparent balance behaviour --- zcash_client_sqlite/src/wallet.rs | 146 +++++++++++++++++++++++++++++- 1 file changed, 144 insertions(+), 2 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index c1872ab477..d33eeb2ec8 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1918,12 +1918,21 @@ mod tests { #[cfg(feature = "transparent-inputs")] use { + crate::testing::{AddressType, TestState}, zcash_client_backend::{ - data_api::WalletWrite, encoding::AddressCodec, wallet::WalletTransparentOutput, + data_api::{wallet::input_selection::GreedyInputSelector, WalletWrite}, + encoding::AddressCodec, + fees::{fixed, DustOutputPolicy}, + wallet::WalletTransparentOutput, }, zcash_primitives::{ consensus::BlockHeight, - transaction::components::{Amount, OutPoint, TxOut}, + memo::MemoBytes, + transaction::{ + components::{amount::NonNegativeAmount, Amount, OutPoint, TxOut}, + fees::fixed::FeeRule as FixedFeeRule, + }, + zip32::ExtendedSpendingKey, }, }; @@ -2046,4 +2055,137 @@ mod tests { let res2 = st.wallet_mut().put_received_transparent_utxo(&utxo2); assert_matches!(res2, Err(_)); } + + #[test] + #[cfg(feature = "transparent-inputs")] + fn transparent_balance_across_shielding() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account_id, usk, _) = st.test_account().unwrap(); + let uaddr = st + .wallet() + .get_current_address(account_id) + .unwrap() + .unwrap(); + let taddr = uaddr.transparent().unwrap(); + + // Initialize the wallet with chain data that has no shielded notes for us. + let not_our_key = ExtendedSpendingKey::master(&[]).to_diversifiable_full_viewing_key(); + let not_our_value = Amount::const_from_i64(10000); + let (start_height, _, _) = + st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); + for _ in 1..10 { + st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); + } + st.scan_cached_blocks(start_height, 10); + + let check_balance = |st: &TestState<_>, min_confirmations: u32, expected| { + // Check the wallet summary returns the expected transparent balance. + let summary = st + .wallet() + .get_wallet_summary(min_confirmations) + .unwrap() + .unwrap(); + let balance = summary.account_balances().get(&account_id).unwrap(); + assert_eq!(balance.unshielded, expected); + + // Check the older APIs for consistency. + let max_height = st.wallet().chain_height().unwrap().unwrap() + 1 - min_confirmations; + assert_eq!( + st.wallet() + .get_transparent_balances(account_id, max_height) + .unwrap() + .get(taddr) + .cloned() + .unwrap_or(Amount::zero()), + Amount::from(expected), + ); + assert_eq!( + st.wallet() + .get_unspent_transparent_outputs(taddr, max_height, &[]) + .unwrap() + .into_iter() + .map(|utxo| utxo.value()) + .sum::>(), + Some(Amount::from(expected)), + ); + }; + + // The wallet starts out with zero balance. + check_balance(&st, 0, NonNegativeAmount::ZERO); + check_balance(&st, 1, NonNegativeAmount::ZERO); + + // Create a fake transparent output. + let value = NonNegativeAmount::from_u64(100000).unwrap(); + let outpoint = OutPoint::new([1u8; 32], 1); + let txout = TxOut { + value: value.into(), + script_pubkey: taddr.script(), + }; + + // Pretend the output was received in the chain tip. + let height = st.wallet().chain_height().unwrap().unwrap(); + let utxo = WalletTransparentOutput::from_parts(outpoint, txout, height).unwrap(); + st.wallet_mut() + .put_received_transparent_utxo(&utxo) + .unwrap(); + + // The wallet should detect the balance as having 1 confirmation. + check_balance(&st, 0, value); + check_balance(&st, 1, value); + check_balance(&st, 2, NonNegativeAmount::ZERO); + + // Shield the output. + let input_selector = GreedyInputSelector::new( + fixed::SingleOutputChangeStrategy::new(FixedFeeRule::non_standard(Amount::zero())), + DustOutputPolicy::default(), + ); + let txid = st + .shield_transparent_funds( + &input_selector, + value, + &usk, + &[*taddr], + &MemoBytes::empty(), + NonZeroU32::new(1).unwrap(), + ) + .unwrap(); + + // The wallet should have zero transparent balance, because the shielding + // transaction can be mined. + check_balance(&st, 0, NonNegativeAmount::ZERO); + check_balance(&st, 1, NonNegativeAmount::ZERO); + check_balance(&st, 2, NonNegativeAmount::ZERO); + + // Mine the shielding transaction. + let (mined_height, _) = st.generate_next_block_including(txid); + st.scan_cached_blocks(mined_height, 1); + + // The wallet should still have zero transparent balance. + check_balance(&st, 0, NonNegativeAmount::ZERO); + check_balance(&st, 1, NonNegativeAmount::ZERO); + check_balance(&st, 2, NonNegativeAmount::ZERO); + + // Unmine the shielding transaction via a reorg. + st.wallet_mut() + .truncate_to_height(mined_height - 1) + .unwrap(); + + // The wallet should still have zero transparent balance. + check_balance(&st, 0, NonNegativeAmount::ZERO); + check_balance(&st, 1, NonNegativeAmount::ZERO); + check_balance(&st, 2, NonNegativeAmount::ZERO); + + // Expire the shielding transaction. + let expiry_height = st.wallet().get_transaction(txid).unwrap().expiry_height(); + st.wallet_mut().update_chain_tip(expiry_height).unwrap(); + + // The transparent output should be spendable again, with more confirmations. + check_balance(&st, 0, value); + check_balance(&st, 1, value); + check_balance(&st, 2, value); + } } From 625a5ff59450d4cd48e6b5e121e927e36e962b90 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 22 Sep 2023 01:19:49 +0000 Subject: [PATCH 0195/1122] zcash_client_sqlite: Remove is-mined checks from transparent balance The `LEFT OUTER JOIN` was causing the `tx.block IS NULL` check to alias two cases: an unspent transparent output, and a transparent output spent in an unmined transaction. The latter only makes sense to include in the UTXO count if the transaction is expired, and (due to limitations of the transparent data model in the current wallet) if that expiry won't be undone by a reorg. We now handle these two cases directly. Partly reverts 8828276361e02155f49da31d2cf9515231f4d26b. Closes zcash/librustzcash#983. Co-authored-by: Kris Nuttycombe --- zcash_client_sqlite/src/wallet.rs | 50 +++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index d33eeb2ec8..69fa9e6e3b 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -713,17 +713,21 @@ pub(crate) fn get_wallet_summary( #[cfg(feature = "transparent-inputs")] { let zero_conf_height = (chain_tip_height + 1).saturating_sub(min_confirmations); + let stable_height = chain_tip_height.saturating_sub(PRUNING_DEPTH); + let mut stmt_transparent_balances = conn.prepare( "SELECT u.received_by_account, SUM(u.value_zat) FROM utxos u LEFT OUTER JOIN transactions tx ON tx.id_tx = u.spent_in_tx WHERE u.height <= :max_height - AND tx.block IS NULL + AND (u.spent_in_tx IS NULL OR (tx.block IS NULL AND tx.expiry_height <= :stable_height)) GROUP BY u.received_by_account", )?; - let mut rows = stmt_transparent_balances - .query(named_params![":max_height": u32::from(zero_conf_height)])?; + let mut rows = stmt_transparent_balances.query(named_params![ + ":max_height": u32::from(zero_conf_height), + ":stable_height": u32::from(stable_height) + ])?; while let Some(row) = rows.next()? { let account = AccountId::from(row.get::<_, u32>(0)?); @@ -1302,15 +1306,20 @@ pub(crate) fn get_unspent_transparent_outputs( max_height: BlockHeight, exclude: &[OutPoint], ) -> Result, SqliteClientError> { + let chain_tip_height = scan_queue_extrema(conn)?.map(|(_, max)| max); + let stable_height = chain_tip_height + .unwrap_or(max_height) + .saturating_sub(PRUNING_DEPTH); + let mut stmt_blocks = conn.prepare( "SELECT u.prevout_txid, u.prevout_idx, u.script, - u.value_zat, u.height, tx.block as block + u.value_zat, u.height FROM utxos u LEFT OUTER JOIN transactions tx ON tx.id_tx = u.spent_in_tx WHERE u.address = :address AND u.height <= :max_height - AND tx.block IS NULL", + AND (u.spent_in_tx IS NULL OR (tx.block IS NULL AND tx.expiry_height <= :stable_height))", )?; let addr_str = address.encode(params); @@ -1318,7 +1327,8 @@ pub(crate) fn get_unspent_transparent_outputs( let mut utxos = Vec::::new(); let mut rows = stmt_blocks.query(named_params![ ":address": addr_str, - ":max_height": u32::from(max_height) + ":max_height": u32::from(max_height), + ":stable_height": u32::from(stable_height), ])?; let excluded: BTreeSet = exclude.iter().cloned().collect(); while let Some(row) = rows.next()? { @@ -1367,6 +1377,11 @@ pub(crate) fn get_transparent_balances( account: AccountId, max_height: BlockHeight, ) -> Result, SqliteClientError> { + let chain_tip_height = scan_queue_extrema(conn)?.map(|(_, max)| max); + let stable_height = chain_tip_height + .unwrap_or(max_height) + .saturating_sub(PRUNING_DEPTH); + let mut stmt_blocks = conn.prepare( "SELECT u.address, SUM(u.value_zat) FROM utxos u @@ -1374,14 +1389,15 @@ pub(crate) fn get_transparent_balances( ON tx.id_tx = u.spent_in_tx WHERE u.received_by_account = :account_id AND u.height <= :max_height - AND tx.block IS NULL + AND (u.spent_in_tx IS NULL OR (tx.block IS NULL AND tx.expiry_height <= :stable_height)) GROUP BY u.address", )?; let mut res = HashMap::new(); let mut rows = stmt_blocks.query(named_params![ ":account_id": u32::from(account), - ":max_height": u32::from(max_height) + ":max_height": u32::from(max_height), + ":stable_height": u32::from(stable_height), ])?; while let Some(row) = rows.next()? { let taddr_str: String = row.get(0)?; @@ -1918,7 +1934,10 @@ mod tests { #[cfg(feature = "transparent-inputs")] use { - crate::testing::{AddressType, TestState}, + crate::{ + testing::{AddressType, TestState}, + PRUNING_DEPTH, + }, zcash_client_backend::{ data_api::{wallet::input_selection::GreedyInputSelector, WalletWrite}, encoding::AddressCodec, @@ -2183,6 +2202,19 @@ mod tests { let expiry_height = st.wallet().get_transaction(txid).unwrap().expiry_height(); st.wallet_mut().update_chain_tip(expiry_height).unwrap(); + // TODO: Making the transparent output spendable in this situation requires + // changes to the transparent data model, so for now the wallet should still have + // zero transparent balance. https://github.com/zcash/librustzcash/issues/986 + check_balance(&st, 0, NonNegativeAmount::ZERO); + check_balance(&st, 1, NonNegativeAmount::ZERO); + check_balance(&st, 2, NonNegativeAmount::ZERO); + + // Roll forward the chain tip until the transaction's expiry height is in the + // stable block range (so a reorg won't make it spendable again). + st.wallet_mut() + .update_chain_tip(expiry_height + PRUNING_DEPTH) + .unwrap(); + // The transparent output should be spendable again, with more confirmations. check_balance(&st, 0, value); check_balance(&st, 1, value); From 513abf8b9758ca7062a6d80c45c55ff7829a8eae Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 22 Sep 2023 01:24:03 +0000 Subject: [PATCH 0196/1122] rustfmt --- zcash_client_backend/src/data_api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 18646c44d1..875ac614b3 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -490,7 +490,7 @@ impl ScannedBlock { } /// Returns the metadata describing the state of the note commitment trees as of the end of the - /// scanned block. + /// scanned block. /// /// The metadata returned from this method is guaranteed to be consistent with what is returned /// by [`Self::height`] and [`Self::block_hash`]. From b76b028b3a84d58318a9f6433f0d3ff95468ac6b Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 22 Sep 2023 15:04:51 +0000 Subject: [PATCH 0197/1122] zcash_client_sqlite: Set chain tip to truncation height when truncating We don't know at truncation time what the latest chain tip is; the chain might have reorged to a shorter heavier chain, or the reorg depth might only be a few blocks. `WalletDb::chain_height` uses the scan queue as its source of truth, so the `Verify` range we add during truncation (to prioritise determining whether the rewind was sufficient) can't extend beyond the block height we know to exist. The next call to `WalletDb::update_chain_tip` will add additional ranges beyond this height, which might include a `Verify` range that ends up merging with the one added during truncation. --- zcash_client_sqlite/src/wallet.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 69fa9e6e3b..70f52d97bf 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -105,8 +105,8 @@ use zcash_client_backend::{ use crate::wallet::commitment_tree::SqliteShardStore; use crate::{ error::SqliteClientError, SqlTransaction, WalletCommitmentTrees, WalletDb, PRUNING_DEPTH, + SAPLING_TABLES_PREFIX, }; -use crate::{SAPLING_TABLES_PREFIX, VERIFY_LOOKAHEAD}; use self::scanning::{parse_priority_code, replace_queue_entries}; @@ -1281,8 +1281,8 @@ pub(crate) fn truncate_to_height( named_params![":end_height": u32::from(block_height + 1)], )?; - // Prioritize the range starting at the height we just rewound to for verification - let query_range = block_height..(block_height + VERIFY_LOOKAHEAD); + // Prioritize the height we just rewound to for verification. + let query_range = block_height..(block_height + 1); let scan_range = ScanRange::from_parts(query_range.clone(), ScanPriority::Verify); replace_queue_entries::( conn, @@ -2192,6 +2192,7 @@ mod tests { st.wallet_mut() .truncate_to_height(mined_height - 1) .unwrap(); + assert_eq!(st.wallet().chain_height().unwrap(), Some(mined_height - 1)); // The wallet should still have zero transparent balance. check_balance(&st, 0, NonNegativeAmount::ZERO); From 4afc24af6a002a0fabd079983acddc900db32d07 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 22 Sep 2023 23:25:19 +0100 Subject: [PATCH 0198/1122] zcash_client_backend 0.10.0-rc.4 --- Cargo.lock | 2 +- zcash_client_backend/CHANGELOG.md | 9 ++++++++- zcash_client_backend/Cargo.toml | 2 +- zcash_client_sqlite/Cargo.toml | 4 ++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2392353e2..d327667f76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2946,7 +2946,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" -version = "0.10.0-rc.3" +version = "0.10.0-rc.4" dependencies = [ "assert_matches", "base64", diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 2280d6f859..c7b0170391 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -7,7 +7,7 @@ and this library adheres to Rust's notion of ## [Unreleased] -## [0.10.0-rc.3] - 2023-09-19 +## [0.10.0-rc.4] - 2023-09-22 ### Notable Changes - `zcash_client_backend` now supports out-of-order scanning of blockchain history. @@ -48,6 +48,10 @@ and this library adheres to Rust's notion of - `testing::MockWalletDb::new` - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}` - `SAPLING_SHARD_HEIGHT` constant +- `zcash_client_backend::proto::compact_formats`: + - `impl From<&sapling::SpendDescription> for CompactSaplingSpend` + - `impl From<&sapling::OutputDescription> for CompactSaplingOutput` + - `impl From<&orchard::Action> for CompactOrchardAction` - `zcash_client_backend::wallet::WalletSaplingOutput::note_commitment_tree_position` - `zcash_client_backend::scanning`: - `ScanError` @@ -143,6 +147,9 @@ and this library adheres to Rust's notion of `chain::scan_cached_blocks`. - `zcash_client_backend::data_api::chain::error::{ChainError, Cause}` have been replaced by `zcash_client_backend::scanning::ScanError` +- `zcash_client_backend::proto::compact_formats`: + - `impl From> for CompactSaplingOutput` + (use `From<&sapling::OutputDescription>` instead). - `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}` have been removed as individual incremental witnesses are no longer tracked on a per-note basis. The global note commitment tree for the wallet should be used diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 5e90c52655..1e19c44136 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_backend" description = "APIs for creating shielded Zcash light clients" -version = "0.10.0-rc.3" +version = "0.10.0-rc.4" authors = [ "Jack Grigg ", "Kris Nuttycombe " diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 16368a8543..ce5f0d9759 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -14,7 +14,7 @@ edition = "2021" rust-version = "1.65" [dependencies] -zcash_client_backend = { version = "=0.10.0-rc.3", path = "../zcash_client_backend", features = ["unstable-serialization", "unstable-spanning-tree"]} +zcash_client_backend = { version = "=0.10.0-rc.4", path = "../zcash_client_backend", features = ["unstable-serialization", "unstable-spanning-tree"]} zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false } @@ -62,7 +62,7 @@ tempfile = "3.5.0" zcash_note_encryption = "0.4" zcash_proofs = { version = "=0.13.0-rc.1", path = "../zcash_proofs" } zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", features = ["test-dependencies"] } -zcash_client_backend = { version = "=0.10.0-rc.3", path = "../zcash_client_backend", features = ["test-dependencies", "unstable-serialization", "unstable-spanning-tree"] } +zcash_client_backend = { version = "=0.10.0-rc.4", path = "../zcash_client_backend", features = ["test-dependencies", "unstable-serialization", "unstable-spanning-tree"] } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } [features] From 30c58ebfb2f5a2c78e6d94232fed17f0a00fa536 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 22 Sep 2023 23:26:43 +0100 Subject: [PATCH 0199/1122] zcash_client_sqlite 0.8.0-rc.5 --- Cargo.lock | 2 +- zcash_client_sqlite/CHANGELOG.md | 2 +- zcash_client_sqlite/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d327667f76..e0131c32d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2988,7 +2988,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" -version = "0.8.0-rc.4" +version = "0.8.0-rc.5" dependencies = [ "assert_matches", "bs58", diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 957de54a66..9702928f15 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -7,7 +7,7 @@ and this library adheres to Rust's notion of ## [Unreleased] -## [0.8.0-rc.4] - 2023-09-19 +## [0.8.0-rc.5] - 2023-09-22 ### Notable Changes - The `v_transactions` and `v_tx_outputs` views have changed in terms of what diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index ce5f0d9759..6cc1f664a3 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_sqlite" description = "An SQLite-based Zcash light client" -version = "0.8.0-rc.4" +version = "0.8.0-rc.5" authors = [ "Jack Grigg ", "Kris Nuttycombe " From b90f285359ebc691278e451d47ce6ce576bccc20 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Sun, 24 Sep 2023 21:35:04 -0600 Subject: [PATCH 0200/1122] zcash_client_sqlite: Include spent utxos in v_transactions net value. This fixes a bug in v_transactions whereby shielding transactions displayed an incorrect balance. --- zcash_client_sqlite/src/wallet/init.rs | 12 ++ .../src/wallet/init/migrations.rs | 4 + .../v_transactions_shielding_balance.rs | 155 ++++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/v_transactions_shielding_balance.rs diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index e8c4341511..38ab334043 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -478,6 +478,18 @@ mod tests { FROM sapling_received_notes JOIN transactions ON transactions.id_tx = sapling_received_notes.spent + UNION + SELECT utxos.received_by_account AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 0 AS pool, + -utxos.value_zat AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM utxos + JOIN transactions + ON transactions.id_tx = utxos.spent_in_tx ), sent_note_counts AS ( SELECT sent_notes.from_account AS account_id, diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 0deaba2995..7331064beb 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -12,6 +12,7 @@ mod ufvk_support; mod utxos_table; mod v_sapling_shard_unscanned_ranges; mod v_transactions_net; +mod v_transactions_shielding_balance; mod v_transactions_transparent_history; mod v_tx_outputs_use_legacy_false; mod wallet_summaries; @@ -50,6 +51,8 @@ pub(super) fn all_migrations( // v_transactions_transparent_history // | // v_tx_outputs_use_legacy_false + // | + // v_transactions_shielding_balance vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -83,5 +86,6 @@ pub(super) fn all_migrations( Box::new(wallet_summaries::Migration), Box::new(v_transactions_transparent_history::Migration), Box::new(v_tx_outputs_use_legacy_false::Migration), + Box::new(v_transactions_shielding_balance::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_shielding_balance.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_shielding_balance.rs new file mode 100644 index 0000000000..9a8c49cef1 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_shielding_balance.rs @@ -0,0 +1,155 @@ +//! This migration reworks transaction history views to correctly include spent transparent utxo +//! value. + +use std::collections::HashSet; + +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::WalletMigrationError; + +use super::v_tx_outputs_use_legacy_false; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xb8fe5112_4365_473c_8b42_2b07c0f0adaf); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [v_tx_outputs_use_legacy_false::MIGRATION_ID] + .into_iter() + .collect() + } + + fn description(&self) -> &'static str { + "Updates v_transactions to include spent UTXOs." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + transaction.execute_batch( + "DROP VIEW v_transactions; + CREATE VIEW v_transactions AS + WITH + notes AS ( + SELECT sapling_received_notes.account AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 2 AS pool, + sapling_received_notes.value AS value, + CASE + WHEN sapling_received_notes.is_change THEN 1 + ELSE 0 + END AS is_change, + CASE + WHEN sapling_received_notes.is_change THEN 0 + ELSE 1 + END AS received_count, + CASE + WHEN (sapling_received_notes.memo IS NULL OR sapling_received_notes.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present + FROM sapling_received_notes + JOIN transactions + ON transactions.id_tx = sapling_received_notes.tx + UNION + SELECT utxos.received_by_account AS account_id, + utxos.height AS block, + utxos.prevout_txid AS txid, + 0 AS pool, + utxos.value_zat AS value, + 0 AS is_change, + 1 AS received_count, + 0 AS memo_present + FROM utxos + UNION + SELECT sapling_received_notes.account AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 2 AS pool, + -sapling_received_notes.value AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM sapling_received_notes + JOIN transactions + ON transactions.id_tx = sapling_received_notes.spent + UNION + SELECT utxos.received_by_account AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 0 AS pool, + -utxos.value_zat AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM utxos + JOIN transactions + ON transactions.id_tx = utxos.spent_in_tx + ), + sent_note_counts AS ( + SELECT sent_notes.from_account AS account_id, + transactions.txid AS txid, + COUNT(DISTINCT sent_notes.id_note) as sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR sapling_received_notes.tx IS NOT NULL) + THEN 0 + ELSE 1 + END + ) AS memo_count + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN sapling_received_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + WHERE COALESCE(sapling_received_notes.is_change, 0) = 0 + GROUP BY account_id, txid + ), + blocks_max_height AS ( + SELECT MAX(blocks.height) as max_height FROM blocks + ) + SELECT notes.account_id AS account_id, + notes.block AS mined_height, + notes.txid AS txid, + transactions.tx_index AS tx_index, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.is_change) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( + blocks.height IS NULL + AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height + ) AS expired_unmined + FROM notes + LEFT JOIN transactions + ON notes.txid = transactions.txid + JOIN blocks_max_height + LEFT JOIN blocks ON blocks.height = notes.block + LEFT JOIN sent_note_counts + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.txid = notes.txid + GROUP BY notes.account_id, notes.txid;" + )?; + + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + panic!("This migration cannot be reverted."); + } +} From 478b5d1858a1495356390ea302be54fbecc38ea7 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 25 Sep 2023 11:14:24 +0000 Subject: [PATCH 0201/1122] zcash_client_sqlite: Clean up migrations graph comment --- .../src/wallet/init/migrations.rs | 47 +++++++++---------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 7331064beb..db8ee2194d 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -27,32 +27,27 @@ pub(super) fn all_migrations( params: &P, seed: Option>, ) -> Vec>> { - // initial_setup - // / \ - // utxos_table ufvk_support --------- - // \ | \ - // \ addresses_table sent_notes_to_internal - // \ / / - // add_utxo_account / - // \ / - // add_transaction_views - // | - // v_transactions_net - // | - // received_notes_nullable_nf - // / | \ - // shardtree_support nullifier_map sapling_memo_consistency - // | | - // add_account_birthdays | - // | | - // v_sapling_shard_unscanned_ranges | - // | | - // wallet_summaries | - // v_transactions_transparent_history - // | - // v_tx_outputs_use_legacy_false - // | - // v_transactions_shielding_balance + // initial_setup + // / \ + // utxos_table ufvk_support + // | / \ + // | addresses_table sent_notes_to_internal + // | / / + // add_utxo_account / + // \ / + // add_transaction_views + // | + // v_transactions_net + // | + // received_notes_nullable_nf + // / | \ + // shardtree_support nullifier_map sapling_memo_consistency + // | | + // add_account_birthdays v_transactions_transparent_history + // | | + // v_sapling_shard_unscanned_ranges v_tx_outputs_use_legacy_false + // | | + // wallet_summaries v_transactions_shielding_balance vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), From a3332c4267ca5b4fe62f0cdc38f1d88e16854c56 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 25 Sep 2023 11:49:40 +0000 Subject: [PATCH 0202/1122] CI: Add lint to check UUID validity --- .github/workflows/ci.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38f6b56d69..adf34ebc10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,3 +159,35 @@ jobs: - uses: actions/checkout@v4 - name: Check formatting run: cargo fmt --all -- --check + + uuid: + name: UUID validity + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Extract UUIDs + id: extract + run: | + { + echo 'UUIDS<> "$GITHUB_OUTPUT" + - name: Check UUID validity + env: + UUIDS: ${{ steps.extract.outputs.UUIDS }} + run: uuidparse -n -o type $UUIDS | xargs -L 1 test "invalid" != + - name: Check UUID type + env: + UUIDS: ${{ steps.extract.outputs.UUIDS }} + run: uuidparse -n -o type $UUIDS | xargs -L 1 test "random" = + - name: Check UUID uniqueness + env: + UUIDS: ${{ steps.extract.outputs.UUIDS }} + run: > + test $( + uuidparse -n -o uuid $U4 | wc -l + ) -eq $( + uuidparse -n -o uuid $U4 | sort | uniq | wc -l + ) From e926151f8fd685be3057fa8fb5d98c8e770b8ce2 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 25 Sep 2023 12:05:42 +0000 Subject: [PATCH 0203/1122] zcash_client_sqlite: Use `Uuid::from_u128` for all UUIDs Previously we used `Uuid::from_fields` to ensure that the various UUID fields were correctly constructed, but now that we have a CI lint that checks this automatically, we can consistently use `Uuid::from_u128` which is easier to prepare from `uuidgen` output. --- .../src/wallet/init/migrations/add_transaction_views.rs | 7 +------ .../src/wallet/init/migrations/add_utxo_account.rs | 9 +-------- .../src/wallet/init/migrations/addresses_table.rs | 9 +-------- .../src/wallet/init/migrations/initial_setup.rs | 9 +-------- .../wallet/init/migrations/received_notes_nullable_nf.rs | 7 +------ .../src/wallet/init/migrations/sent_notes_to_internal.rs | 9 +-------- .../src/wallet/init/migrations/shardtree_support.rs | 7 +------ .../src/wallet/init/migrations/ufvk_support.rs | 7 +------ .../src/wallet/init/migrations/utxos_table.rs | 7 +------ .../src/wallet/init/migrations/v_transactions_net.rs | 7 +------ 10 files changed, 10 insertions(+), 68 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs index 05ab54aed5..d0b422a022 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs @@ -17,12 +17,7 @@ use zcash_primitives::{ use super::{add_utxo_account, sent_notes_to_internal}; use crate::wallet::init::WalletMigrationError; -pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( - 0x282fad2e, - 0x8372, - 0x4ca0, - b"\x8b\xed\x71\x82\x13\x20\x90\x9f", -); +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x282fad2e_8372_4ca0_8bed_71821320909f); pub(crate) struct Migration; diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs index cc3c61f6ad..304a4490a1 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs @@ -20,14 +20,7 @@ use { }; /// This migration adds an account identifier column to the UTXOs table. -/// -/// 761884d6-30d8-44ef-b204-0b82551c4ca1 -pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( - 0x761884d6, - 0x30d8, - 0x44ef, - b"\xb2\x04\x0b\x82\x55\x1c\x4c\xa1", -); +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x761884d6_30d8_44ef_b204_0b82551c4ca1); pub(super) struct Migration

{ pub(super) _params: P, diff --git a/zcash_client_sqlite/src/wallet/init/migrations/addresses_table.rs b/zcash_client_sqlite/src/wallet/init/migrations/addresses_table.rs index 7601f8d387..9456a4fcce 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/addresses_table.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/addresses_table.rs @@ -16,14 +16,7 @@ use super::ufvk_support; /// The migration that removed the address columns from the `accounts` table, and created /// the `accounts` table. -/// -/// d956978c-9c87-4d6e-815d-fb8f088d094c -pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( - 0xd956978c, - 0x9c87, - 0x4d6e, - b"\x81\x5d\xfb\x8f\x08\x8d\x09\x4c", -); +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xd956978c_9c87_4d6e_815d_fb8f088d094c); pub(crate) struct Migration { pub(crate) params: P, diff --git a/zcash_client_sqlite/src/wallet/init/migrations/initial_setup.rs b/zcash_client_sqlite/src/wallet/init/migrations/initial_setup.rs index 8501368564..4c7387aa64 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/initial_setup.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/initial_setup.rs @@ -9,14 +9,7 @@ use uuid::Uuid; use crate::wallet::init::WalletMigrationError; /// Identifier for the migration that performs the initial setup of the wallet database. -/// -/// bc4f5e57-d600-4b6c-990f-b3538f0bfce1, -pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( - 0xbc4f5e57, - 0xd600, - 0x4b6c, - b"\x99\x0f\xb3\x53\x8f\x0b\xfc\xe1", -); +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xbc4f5e57_d600_4b6c_990f_b3538f0bfce1); pub(super) struct Migration; diff --git a/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs b/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs index 5a856dc3d2..ff7b14e976 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs @@ -11,12 +11,7 @@ use uuid::Uuid; use super::v_transactions_net; use crate::wallet::init::WalletMigrationError; -pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( - 0xbdcdcedc, - 0x7b29, - 0x4f1c, - b"\x83\x07\x35\xf9\x37\xf0\xd3\x2a", -); +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xbdcdcedc_7b29_4f1c_8307_35f937f0d32a); pub(crate) struct Migration; diff --git a/zcash_client_sqlite/src/wallet/init/migrations/sent_notes_to_internal.rs b/zcash_client_sqlite/src/wallet/init/migrations/sent_notes_to_internal.rs index ee191b9570..a647894dc3 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/sent_notes_to_internal.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/sent_notes_to_internal.rs @@ -11,14 +11,7 @@ use super::ufvk_support; use crate::wallet::init::WalletMigrationError; /// This migration adds the `to_account` field to the `sent_notes` table. -/// -/// 0ddbe561-8259-4212-9ab7-66fdc4a74e1d -pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( - 0x0ddbe561, - 0x8259, - 0x4212, - b"\x9a\xb7\x66\xfd\xc4\xa7\x4e\x1d", -); +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x0ddbe561_8259_4212_9ab7_66fdc4a74e1d); pub(super) struct Migration; diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index e35d01805e..822c87d9be 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -32,12 +32,7 @@ use crate::{ PRUNING_DEPTH, SAPLING_TABLES_PREFIX, }; -pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( - 0x7da6489d, - 0xe835, - 0x4657, - b"\x8b\xe5\xf5\x12\xbc\xce\x6c\xbf", -); +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x7da6489d_e835_4657_8be5_f512bcce6cbf); pub(super) struct Migration

{ pub(super) params: P, diff --git a/zcash_client_sqlite/src/wallet/init/migrations/ufvk_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/ufvk_support.rs index 656281b704..889d8d0ae4 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/ufvk_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/ufvk_support.rs @@ -25,12 +25,7 @@ use crate::wallet::{ pool_code, }; -pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( - 0xbe57ef3b, - 0x388e, - 0x42ea, - b"\x97\xe2\x67\x8d\xaf\xcf\x97\x54", -); +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xbe57ef3b_388e_42ea_97e2_678dafcf9754); pub(super) struct Migration

{ pub(super) params: P, diff --git a/zcash_client_sqlite/src/wallet/init/migrations/utxos_table.rs b/zcash_client_sqlite/src/wallet/init/migrations/utxos_table.rs index 7180394292..083393e677 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/utxos_table.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/utxos_table.rs @@ -8,12 +8,7 @@ use uuid::Uuid; use crate::wallet::init::{migrations::initial_setup, WalletMigrationError}; -pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( - 0xa2e0ed2e, - 0x8852, - 0x475e, - b"\xb0\xa4\xf1\x54\xb1\x5b\x9d\xbe", -); +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xa2e0ed2e_8852_475e_b0a4_f154b15b9dbe); pub(super) struct Migration; diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs index 14cd830b13..c6ad76403d 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs @@ -11,12 +11,7 @@ use zcash_client_backend::data_api::{PoolType, ShieldedProtocol}; use super::add_transaction_views; use crate::wallet::{init::WalletMigrationError, pool_code}; -pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( - 0x2aa4d24f, - 0x51aa, - 0x4a4c, - b"\x8d\x9b\xe5\xb8\xa7\x62\x86\x5f", -); +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x2aa4d24f_51aa_4a4c_8d9b_e5b8a762865f); pub(crate) struct Migration; From d35d0961ab61dcc833f902641868aaba164bb9c6 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 25 Sep 2023 15:06:57 +0000 Subject: [PATCH 0204/1122] zcash_primitives 0.13.0 --- Cargo.lock | 2 +- zcash_client_backend/Cargo.toml | 2 +- zcash_client_sqlite/Cargo.toml | 4 ++-- zcash_extensions/Cargo.toml | 2 +- zcash_primitives/CHANGELOG.md | 5 ++--- zcash_primitives/Cargo.toml | 2 +- zcash_proofs/Cargo.toml | 2 +- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e0131c32d0..32cb63f3f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3077,7 +3077,7 @@ dependencies = [ [[package]] name = "zcash_primitives" -version = "0.13.0-rc.1" +version = "0.13.0" dependencies = [ "aes", "assert_matches", diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 1e19c44136..5d275900dd 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -23,7 +23,7 @@ development = ["zcash_proofs"] zcash_address = { version = "0.3", path = "../components/zcash_address" } zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_note_encryption = "0.4" -zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false } +zcash_primitives = { version = "0.13", path = "../zcash_primitives", default-features = false } # Dependencies exposed in a public API: # (Breaking upgrades to these require a breaking upgrade to this crate.) diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 6cc1f664a3..bbe6783e09 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -16,7 +16,7 @@ rust-version = "1.65" [dependencies] zcash_client_backend = { version = "=0.10.0-rc.4", path = "../zcash_client_backend", features = ["unstable-serialization", "unstable-spanning-tree"]} zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } -zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false } +zcash_primitives = { version = "0.13", path = "../zcash_primitives", default-features = false } # Dependencies exposed in a public API: # (Breaking upgrades to these require a breaking upgrade to this crate.) @@ -61,7 +61,7 @@ regex = "1.4" tempfile = "3.5.0" zcash_note_encryption = "0.4" zcash_proofs = { version = "=0.13.0-rc.1", path = "../zcash_proofs" } -zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", features = ["test-dependencies"] } +zcash_primitives = { version = "0.13", path = "../zcash_primitives", features = ["test-dependencies"] } zcash_client_backend = { version = "=0.10.0-rc.4", path = "../zcash_client_backend", features = ["test-dependencies", "unstable-serialization", "unstable-spanning-tree"] } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } diff --git a/zcash_extensions/Cargo.toml b/zcash_extensions/Cargo.toml index 4dfc014d13..d0e345c4ff 100644 --- a/zcash_extensions/Cargo.toml +++ b/zcash_extensions/Cargo.toml @@ -11,7 +11,7 @@ rust-version = "1.65" [dependencies] blake2b_simd = "1" -zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false, features = ["zfuture" ] } +zcash_primitives = { version = "0.13", path = "../zcash_primitives", default-features = false, features = ["zfuture" ] } [dev-dependencies] ff = "0.13" diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index e708c8f36d..b0450f2102 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -6,10 +6,8 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] -### Added -- Trait implementation `Mul` for `NonNegativeAmount`. -## [0.13.0-rc.1] - 2023-09-08 +## [0.13.0] - 2023-09-25 ### Added - `zcash_primitives::consensus::BlockHeight::saturating_sub` - `zcash_primitives::transaction::builder`: @@ -26,6 +24,7 @@ and this library adheres to Rust's notion of - `Add for Option` - `Sub for NonNegativeAmount` - `Sub for Option` + - `Mul for NonNegativeAmount` - `zcash_primitives::block::BlockHash::try_from_slice` ### Changed diff --git a/zcash_primitives/Cargo.toml b/zcash_primitives/Cargo.toml index f438b8083b..e88ed685d3 100644 --- a/zcash_primitives/Cargo.toml +++ b/zcash_primitives/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_primitives" description = "Rust implementations of the Zcash primitives" -version = "0.13.0-rc.1" +version = "0.13.0" authors = [ "Jack Grigg ", "Kris Nuttycombe " diff --git a/zcash_proofs/Cargo.toml b/zcash_proofs/Cargo.toml index d62b5e546e..9f2ca15dc5 100644 --- a/zcash_proofs/Cargo.toml +++ b/zcash_proofs/Cargo.toml @@ -17,7 +17,7 @@ categories = ["cryptography::cryptocurrencies"] all-features = true [dependencies] -zcash_primitives = { version = "=0.13.0-rc.1", path = "../zcash_primitives", default-features = false } +zcash_primitives = { version = "0.13", path = "../zcash_primitives", default-features = false } # Dependencies exposed in a public API: # (Breaking upgrades to these require a breaking upgrade to this crate.) From 1a1abb62aa765f4fae12b968bf17ac135ad60a18 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 25 Sep 2023 15:11:00 +0000 Subject: [PATCH 0205/1122] zcash_proofs 0.13.0 --- Cargo.lock | 2 +- zcash_client_backend/Cargo.toml | 2 +- zcash_client_sqlite/Cargo.toml | 2 +- zcash_extensions/Cargo.toml | 2 +- zcash_proofs/CHANGELOG.md | 2 +- zcash_proofs/Cargo.toml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 32cb63f3f4..c7c5bccea5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3117,7 +3117,7 @@ dependencies = [ [[package]] name = "zcash_proofs" -version = "0.13.0-rc.1" +version = "0.13.0" dependencies = [ "bellman", "blake2b_simd", diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 5d275900dd..37364e3312 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -88,7 +88,7 @@ proptest = "1.0.0" rand_core = "0.6" rand_xorshift = "0.3" tempfile = "3.5.0" -zcash_proofs = { version = "=0.13.0-rc.1", path = "../zcash_proofs", default-features = false } +zcash_proofs = { version = "0.13", path = "../zcash_proofs", default-features = false } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } time = ">=0.3.22, <0.3.24" # time 0.3.24 has MSRV 1.67 diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index bbe6783e09..db294e02c8 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -60,7 +60,7 @@ rand_core = "0.6" regex = "1.4" tempfile = "3.5.0" zcash_note_encryption = "0.4" -zcash_proofs = { version = "=0.13.0-rc.1", path = "../zcash_proofs" } +zcash_proofs = { version = "0.13", path = "../zcash_proofs" } zcash_primitives = { version = "0.13", path = "../zcash_primitives", features = ["test-dependencies"] } zcash_client_backend = { version = "=0.10.0-rc.4", path = "../zcash_client_backend", features = ["test-dependencies", "unstable-serialization", "unstable-spanning-tree"] } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } diff --git a/zcash_extensions/Cargo.toml b/zcash_extensions/Cargo.toml index d0e345c4ff..0123b6e71e 100644 --- a/zcash_extensions/Cargo.toml +++ b/zcash_extensions/Cargo.toml @@ -18,7 +18,7 @@ ff = "0.13" jubjub = "0.10" rand_core = "0.6" zcash_address = { version = "0.3", path = "../components/zcash_address" } -zcash_proofs = { version = "=0.13.0-rc.1", path = "../zcash_proofs" } +zcash_proofs = { version = "0.13", path = "../zcash_proofs" } [features] transparent-inputs = [] diff --git a/zcash_proofs/CHANGELOG.md b/zcash_proofs/CHANGELOG.md index 855a62df03..41872dd149 100644 --- a/zcash_proofs/CHANGELOG.md +++ b/zcash_proofs/CHANGELOG.md @@ -7,7 +7,7 @@ and this library adheres to Rust's notion of ## [Unreleased] -## [0.13.0-rc.1] - 2023-09-08 +## [0.13.0] - 2023-09-25 ### Changed - Bumped dependencies to `zcash_primitives 0.13`. diff --git a/zcash_proofs/Cargo.toml b/zcash_proofs/Cargo.toml index 9f2ca15dc5..a66295fd88 100644 --- a/zcash_proofs/Cargo.toml +++ b/zcash_proofs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_proofs" description = "Zcash zk-SNARK circuits and proving APIs" -version = "0.13.0-rc.1" +version = "0.13.0" authors = [ "Jack Grigg ", ] From 4bc65d66b10e34bc46cab95125e0626a928b9c8b Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 25 Sep 2023 18:19:52 +0000 Subject: [PATCH 0206/1122] zcash_client_backend 0.10.0 --- Cargo.lock | 2 +- zcash_client_backend/CHANGELOG.md | 2 +- zcash_client_backend/Cargo.toml | 2 +- zcash_client_sqlite/Cargo.toml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c7c5bccea5..45f2f7d8aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2946,7 +2946,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" -version = "0.10.0-rc.4" +version = "0.10.0" dependencies = [ "assert_matches", "base64", diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index c7b0170391..aaa3bb93f6 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -7,7 +7,7 @@ and this library adheres to Rust's notion of ## [Unreleased] -## [0.10.0-rc.4] - 2023-09-22 +## [0.10.0] - 2023-09-25 ### Notable Changes - `zcash_client_backend` now supports out-of-order scanning of blockchain history. diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 37364e3312..2f20e040ad 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_backend" description = "APIs for creating shielded Zcash light clients" -version = "0.10.0-rc.4" +version = "0.10.0" authors = [ "Jack Grigg ", "Kris Nuttycombe " diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index db294e02c8..335f97ca56 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -14,7 +14,7 @@ edition = "2021" rust-version = "1.65" [dependencies] -zcash_client_backend = { version = "=0.10.0-rc.4", path = "../zcash_client_backend", features = ["unstable-serialization", "unstable-spanning-tree"]} +zcash_client_backend = { version = "0.10", path = "../zcash_client_backend", features = ["unstable-serialization", "unstable-spanning-tree"]} zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_primitives = { version = "0.13", path = "../zcash_primitives", default-features = false } @@ -62,7 +62,7 @@ tempfile = "3.5.0" zcash_note_encryption = "0.4" zcash_proofs = { version = "0.13", path = "../zcash_proofs" } zcash_primitives = { version = "0.13", path = "../zcash_primitives", features = ["test-dependencies"] } -zcash_client_backend = { version = "=0.10.0-rc.4", path = "../zcash_client_backend", features = ["test-dependencies", "unstable-serialization", "unstable-spanning-tree"] } +zcash_client_backend = { version = "0.10", path = "../zcash_client_backend", features = ["test-dependencies", "unstable-serialization", "unstable-spanning-tree"] } zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } [features] From 7a47f8494c9ff7792bca47883ca612c698c2ee8f Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 25 Sep 2023 18:20:44 +0000 Subject: [PATCH 0207/1122] zcash_client_sqlite 0.8.0 --- Cargo.lock | 2 +- zcash_client_sqlite/CHANGELOG.md | 2 +- zcash_client_sqlite/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 45f2f7d8aa..e594db32ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2988,7 +2988,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" -version = "0.8.0-rc.5" +version = "0.8.0" dependencies = [ "assert_matches", "bs58", diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 9702928f15..2a044fc2b2 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -7,7 +7,7 @@ and this library adheres to Rust's notion of ## [Unreleased] -## [0.8.0-rc.5] - 2023-09-22 +## [0.8.0] - 2023-09-25 ### Notable Changes - The `v_transactions` and `v_tx_outputs` views have changed in terms of what diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 335f97ca56..bb2717717f 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_sqlite" description = "An SQLite-based Zcash light client" -version = "0.8.0-rc.5" +version = "0.8.0" authors = [ "Jack Grigg ", "Kris Nuttycombe " From 7e89300db909cac0617cf0c306f47ee5f5cd3a2c Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 26 Sep 2023 20:12:23 +0000 Subject: [PATCH 0208/1122] Move common package and dependency configs into workspace config The MSRV for the main crates is 1.65, which is higher than the Rust version that stabilised workplace dependencies (1.64). The implicit MSRV for the component crates is still lower than this, so we don't migrate these crates. --- Cargo.lock | 2 - Cargo.toml | 91 +++++++++++++++++++++ zcash_client_backend/Cargo.toml | 75 +++++++++-------- zcash_client_backend/src/data_api/wallet.rs | 1 - zcash_client_sqlite/Cargo.toml | 57 ++++++------- zcash_extensions/Cargo.toml | 23 +++--- zcash_history/Cargo.toml | 20 ++--- zcash_history/src/version.rs | 2 +- zcash_primitives/Cargo.toml | 78 +++++++++--------- zcash_proofs/Cargo.toml | 32 ++++---- 10 files changed, 235 insertions(+), 146 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e594db32ce..9fb8e2c0c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2968,12 +2968,10 @@ dependencies = [ "proptest", "prost", "rand_core", - "rand_xorshift", "rayon", "secrecy", "shardtree", "subtle", - "tempfile", "time", "tonic", "tonic-build", diff --git a/Cargo.toml b/Cargo.toml index 044d879e93..bb3cf1231f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,97 @@ members = [ "zcash_proofs", ] +[workspace.package] +edition = "2021" +rust-version = "1.65" +repository = "https://github.com/zcash/librustzcash" +license = "MIT OR Apache-2.0" +categories = ["cryptography::cryptocurrencies"] + +# Common dependencies across all of our crates. Dependencies used only by a single crate +# (and that don't have cross-crate versioning needs) are specified by the crate itself. +# +# See the individual crate `Cargo.toml` files for information about which dependencies are +# part of a public API, and which can be updated without a SemVer bump. +[workspace.dependencies] +# Intra-workspace dependencies +equihash = { version = "0.2", path = "components/equihash" } +zcash_address = { version = "0.3", path = "components/zcash_address" } +zcash_client_backend = { version = "0.10", path = "zcash_client_backend" } +zcash_encoding = { version = "0.2", path = "components/zcash_encoding" } +zcash_note_encryption = "0.4" +zcash_primitives = { version = "0.13", path = "zcash_primitives", default-features = false } +zcash_proofs = { version = "0.13", path = "zcash_proofs", default-features = false } + +# Shielded protocols +ff = "0.13" +group = "0.13" +incrementalmerkletree = "0.5" +shardtree = "0.1" + +# Payment protocols +# - Sapling +bitvec = "1" +blake2s_simd = "1" +bls12_381 = "0.8" +jubjub = "0.10" + +# - Orchard +nonempty = "0.7" +orchard = { version = "0.6", default-features = false } + +# - Transparent +hdwallet = "0.4" +ripemd = "0.1" +secp256k1 = "0.26" + +# CSPRNG +rand = "0.8" +rand_core = "0.6" + +# Digests +blake2b_simd = "1" +sha2 = "0.10" + +# Encodings +base64 = "0.21" +bech32 = "0.9" +bs58 = { version = "0.5", features = ["check"] } +byteorder = "1" +hex = "0.4" +percent-encoding = "2.1.0" + +# Logging and metrics +memuse = "0.2.1" +tracing = "0.1" + +# Parallel processing +crossbeam-channel = "0.5" +maybe-rayon = { version = "0.1.0", default-features = false } +rayon = "1.5" + +# Protobuf and gRPC +prost = "0.12" +tonic = "0.10" +tonic-build = "0.10" + +# Secret management +secrecy = "0.8" +subtle = "2.2.3" + +# Static constants +lazy_static = "1" + +# Tests and benchmarks +assert_matches = "1.5" +criterion = "0.4" +proptest = "1" +rand_xorshift = "0.3" + +# ZIP 32 +aes = "0.8" +fpe = "0.6" + [profile.release] lto = true panic = 'abort' diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 2f20e040ad..2522f80efd 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -7,11 +7,12 @@ authors = [ "Kris Nuttycombe " ] homepage = "https://github.com/zcash/librustzcash" -repository = "https://github.com/zcash/librustzcash" +repository.workspace = true readme = "README.md" -license = "MIT OR Apache-2.0" -edition = "2021" -rust-version = "1.65" +license.workspace = true +edition.workspace = true +rust-version.workspace = true +categories.workspace = true # Exclude proto files so crates.io consumers don't need protoc. exclude = ["*.proto"] @@ -20,10 +21,10 @@ exclude = ["*.proto"] development = ["zcash_proofs"] [dependencies] -zcash_address = { version = "0.3", path = "../components/zcash_address" } -zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } -zcash_note_encryption = "0.4" -zcash_primitives = { version = "0.13", path = "../zcash_primitives", default-features = false } +zcash_address.workspace = true +zcash_encoding.workspace = true +zcash_note_encryption.workspace = true +zcash_primitives.workspace = true # Dependencies exposed in a public API: # (Breaking upgrades to these require a breaking upgrade to this crate.) @@ -31,37 +32,37 @@ zcash_primitives = { version = "0.13", path = "../zcash_primitives", default-fea time = "0.3.22" # - Encodings -base64 = "0.21" -bech32 = "0.9" -bs58 = { version = "0.5", features = ["check"] } +base64.workspace = true +bech32.workspace = true +bs58.workspace = true # - Errors -hdwallet = { version = "0.4", optional = true } +hdwallet = { workspace = true, optional = true } # - Logging and metrics -memuse = "0.2" -tracing = "0.1" +memuse.workspace = true +tracing.workspace = true # - Protobuf interfaces and gRPC bindings -hex = "0.4" -prost = "0.12" -tonic = { version = "0.10", optional = true } +hex.workspace = true +prost.workspace = true +tonic = { workspace = true, optional = true } # - Secret management -secrecy = "0.8" -subtle = "2.2.3" +secrecy.workspace = true +subtle.workspace = true # - Shielded protocols -bls12_381 = "0.8" -group = "0.13" -orchard = { version = "0.6", default-features = false } +bls12_381.workspace = true +group.workspace = true +orchard.workspace = true # - Note commitment trees -incrementalmerkletree = "0.5" -shardtree = "0.1" +incrementalmerkletree.workspace = true +shardtree.workspace = true # - Test dependencies -proptest = { version = "1.0.0", optional = true } +proptest = { workspace = true, optional = true } # - ZIP 321 nom = "7" @@ -69,27 +70,25 @@ nom = "7" # Dependencies used internally: # (Breaking upgrades to these are usually backwards-compatible, but check MSRVs.) # - Encodings -byteorder = { version = "1", optional = true } -percent-encoding = "2.1.0" +byteorder = { workspace = true, optional = true } +percent-encoding.workspace = true # - Scanning -crossbeam-channel = "0.5" -rayon = "1.5" +crossbeam-channel.workspace = true +rayon.workspace = true [build-dependencies] -tonic-build = "0.10" +tonic-build.workspace = true which = "4" [dev-dependencies] -assert_matches = "1.5" +assert_matches.workspace = true gumdrop = "0.8" -jubjub = "0.10" -proptest = "1.0.0" -rand_core = "0.6" -rand_xorshift = "0.3" -tempfile = "3.5.0" -zcash_proofs = { version = "0.13", path = "../zcash_proofs", default-features = false } -zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } +jubjub.workspace = true +proptest.workspace = true +rand_core.workspace = true +zcash_proofs.workspace = true +zcash_address = { workspace = true, features = ["test-dependencies"] } time = ">=0.3.22, <0.3.24" # time 0.3.24 has MSRV 1.67 diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 71d0503694..0d1adda6fe 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -127,7 +127,6 @@ where /// ``` /// # #[cfg(all(feature = "test-dependencies", feature = "local-prover"))] /// # { -/// use tempfile::NamedTempFile; /// use zcash_primitives::{ /// consensus::{self, Network}, /// constants::testnet::COIN_TYPE, diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index bb2717717f..a0a9009764 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -7,38 +7,39 @@ authors = [ "Kris Nuttycombe " ] homepage = "https://github.com/zcash/librustzcash" -repository = "https://github.com/zcash/librustzcash" +repository.workspace = true readme = "README.md" -license = "MIT OR Apache-2.0" -edition = "2021" -rust-version = "1.65" +license.workspace = true +edition.workspace = true +rust-version.workspace = true +categories.workspace = true [dependencies] -zcash_client_backend = { version = "0.10", path = "../zcash_client_backend", features = ["unstable-serialization", "unstable-spanning-tree"]} -zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } -zcash_primitives = { version = "0.13", path = "../zcash_primitives", default-features = false } +zcash_client_backend = { workspace = true, features = ["unstable-serialization", "unstable-spanning-tree"] } +zcash_encoding.workspace = true +zcash_primitives.workspace = true # Dependencies exposed in a public API: # (Breaking upgrades to these require a breaking upgrade to this crate.) # - Errors -bs58 = { version = "0.5", features = ["check"] } -hdwallet = { version = "0.4", optional = true } +bs58.workspace = true +hdwallet = { workspace = true, optional = true } # - Logging and metrics -tracing = "0.1" +tracing.workspace = true # - Serialization -byteorder = "1" -prost = "0.12" -group = "0.13" -jubjub = "0.10" +byteorder.workspace = true +prost.workspace = true +group.workspace = true +jubjub.workspace = true # - Secret management -secrecy = "0.8" +secrecy.workspace = true # - Note commitment trees -incrementalmerkletree = "0.5" -shardtree = { version = "0.1", features = ["legacy-api"] } +incrementalmerkletree.workspace = true +shardtree = { workspace = true, features = ["legacy-api"] } # - SQLite databases rusqlite = { version = "0.29.0", features = ["bundled", "time", "array"] } @@ -49,21 +50,21 @@ uuid = "1.1" # Dependencies used internally: # (Breaking upgrades to these are usually backwards-compatible, but check MSRVs.) -maybe-rayon = {version = "0.1.0", default-features = false} +maybe-rayon.workspace = true [dev-dependencies] -assert_matches = "1.5" -incrementalmerkletree = { version = "0.5", features = ["test-dependencies"] } -shardtree = { version = "0.1", features = ["legacy-api", "test-dependencies"] } -proptest = "1.0.0" -rand_core = "0.6" +assert_matches.workspace = true +incrementalmerkletree = { workspace = true, features = ["test-dependencies"] } +shardtree = { workspace = true, features = ["legacy-api", "test-dependencies"] } +proptest.workspace = true +rand_core.workspace = true regex = "1.4" tempfile = "3.5.0" -zcash_note_encryption = "0.4" -zcash_proofs = { version = "0.13", path = "../zcash_proofs" } -zcash_primitives = { version = "0.13", path = "../zcash_primitives", features = ["test-dependencies"] } -zcash_client_backend = { version = "0.10", path = "../zcash_client_backend", features = ["test-dependencies", "unstable-serialization", "unstable-spanning-tree"] } -zcash_address = { version = "0.3", path = "../components/zcash_address", features = ["test-dependencies"] } +zcash_note_encryption.workspace = true +zcash_proofs.workspace = true +zcash_primitives = { workspace = true, features = ["test-dependencies"] } +zcash_client_backend = { workspace = true, features = ["test-dependencies", "unstable-serialization", "unstable-spanning-tree"] } +zcash_address = { workspace = true, features = ["test-dependencies"] } [features] default = ["multicore"] diff --git a/zcash_extensions/Cargo.toml b/zcash_extensions/Cargo.toml index 0123b6e71e..1bd361cafc 100644 --- a/zcash_extensions/Cargo.toml +++ b/zcash_extensions/Cargo.toml @@ -4,21 +4,22 @@ description = "Zcash Extension implementations & consensus node integration laye version = "0.0.0" authors = ["Jack Grigg ", "Kris Nuttycombe "] homepage = "https://github.com/zcash/librustzcash" -repository = "https://github.com/zcash/librustzcash" -license = "MIT OR Apache-2.0" -edition = "2021" -rust-version = "1.65" +repository.workspace = true +license.workspace = true +edition.workspace = true +rust-version.workspace = true +categories.workspace = true [dependencies] -blake2b_simd = "1" -zcash_primitives = { version = "0.13", path = "../zcash_primitives", default-features = false, features = ["zfuture" ] } +blake2b_simd.workspace = true +zcash_primitives = { workspace = true, features = ["zfuture" ] } [dev-dependencies] -ff = "0.13" -jubjub = "0.10" -rand_core = "0.6" -zcash_address = { version = "0.3", path = "../components/zcash_address" } -zcash_proofs = { version = "0.13", path = "../zcash_proofs" } +ff.workspace = true +jubjub.workspace = true +rand_core.workspace = true +zcash_address.workspace = true +zcash_proofs.workspace = true [features] transparent-inputs = [] diff --git a/zcash_history/Cargo.toml b/zcash_history/Cargo.toml index c9de761fb9..a14ae93b5c 100644 --- a/zcash_history/Cargo.toml +++ b/zcash_history/Cargo.toml @@ -2,22 +2,22 @@ name = "zcash_history" version = "0.3.0" authors = ["NikVolf "] -edition = "2021" -rust-version = "1.65" -license = "MIT/Apache-2.0" -documentation = "https://docs.rs/zcash_history/" +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true description = "Library for Zcash blockchain history tools" -categories = ["cryptography::cryptocurrencies"] +categories.workspace = true [dev-dependencies] -assert_matches = "1.3.0" -proptest = "1.0.0" +assert_matches.workspace = true +proptest.workspace = true [dependencies] primitive-types = { version = "0.12", default-features = false } -byteorder = "1" -blake2 = { package = "blake2b_simd", version = "1" } -proptest = { version = "1.0.0", optional = true } +byteorder.workspace = true +blake2b_simd.workspace = true +proptest = { workspace = true, optional = true } [features] test-dependencies = ["proptest"] diff --git a/zcash_history/src/version.rs b/zcash_history/src/version.rs index c9e53157d8..bfc18fa6f0 100644 --- a/zcash_history/src/version.rs +++ b/zcash_history/src/version.rs @@ -1,7 +1,7 @@ use std::fmt; use std::io; -use blake2::Params as Blake2Params; +use blake2b_simd::Params as Blake2Params; use byteorder::{ByteOrder, LittleEndian}; use crate::{node_data, NodeData, MAX_NODE_DATA_SIZE}; diff --git a/zcash_primitives/Cargo.toml b/zcash_primitives/Cargo.toml index e88ed685d3..efcaa9b55d 100644 --- a/zcash_primitives/Cargo.toml +++ b/zcash_primitives/Cargo.toml @@ -7,59 +7,59 @@ authors = [ "Kris Nuttycombe " ] homepage = "https://github.com/zcash/librustzcash" -repository = "https://github.com/zcash/librustzcash" +repository.workspace = true readme = "README.md" -license = "MIT OR Apache-2.0" -edition = "2021" -rust-version = "1.65" -categories = ["cryptography::cryptocurrencies"] +license.workspace = true +edition.workspace = true +rust-version.workspace = true +categories.workspace = true [package.metadata.docs.rs] all-features = true [dependencies] -equihash = { version = "0.2", path = "../components/equihash" } -zcash_address = { version = "0.3", path = "../components/zcash_address" } -zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } +equihash.workspace = true +zcash_address.workspace = true +zcash_encoding.workspace = true # Dependencies exposed in a public API: # (Breaking upgrades to these require a breaking upgrade to this crate.) # - CSPRNG -rand = "0.8" -rand_core = "0.6" +rand.workspace = true +rand_core.workspace = true # - Digests (output types exposed) -blake2b_simd = "1" -sha2 = "0.10" +blake2b_simd.workspace = true +sha2.workspace = true # - Metrics -memuse = "0.2.1" +memuse.workspace = true # - Secret management -subtle = "2.2.3" +subtle.workspace = true # - Shielded protocols -bls12_381 = "0.8" -ff = "0.13" -group = { version = "0.13", features = ["wnaf-memuse"] } -jubjub = "0.10" -nonempty = "0.7" -orchard = { version = "0.6", default-features = false } +bls12_381.workspace = true +ff.workspace = true +group = { workspace = true, features = ["wnaf-memuse"] } +jubjub.workspace = true +nonempty.workspace = true +orchard.workspace = true # - Note Commitment Trees -incrementalmerkletree = { version = "0.5", features = ["legacy-api"] } +incrementalmerkletree = { workspace = true, features = ["legacy-api"] } # - Static constants -lazy_static = "1" +lazy_static.workspace = true # - Test dependencies -proptest = { version = "1.0.0", optional = true } +proptest = { workspace = true, optional = true } # - Transparent inputs # - `Error` type exposed -hdwallet = { version = "0.4", optional = true } +hdwallet = { workspace = true, optional = true } # - `SecretKey` and `PublicKey` types exposed -secp256k1 = { version = "0.26", optional = true } +secp256k1 = { workspace = true, optional = true } # - ZIP 339 bip0039 = { version = "0.10", features = ["std", "all-languages"] } @@ -67,32 +67,32 @@ bip0039 = { version = "0.10", features = ["std", "all-languages"] } # Dependencies used internally: # (Breaking upgrades to these are usually backwards-compatible, but check MSRVs.) # - Encodings -byteorder = "1" -hex = "0.4" +byteorder.workspace = true +hex.workspace = true # - Shielded protocols -bitvec = "1" -blake2s_simd = "1" +bitvec.workspace = true +blake2s_simd.workspace = true # - Transparent inputs -ripemd = { version = "0.1", optional = true } +ripemd = { workspace = true, optional = true } # - ZIP 32 -aes = "0.8" -fpe = "0.6" +aes.workspace = true +fpe.workspace = true [dependencies.zcash_note_encryption] -version = "0.4" +workspace = true features = ["pre-zip-212"] [dev-dependencies] chacha20poly1305 = "0.10" -criterion = "0.4" -incrementalmerkletree = { version = "0.5", features = ["legacy-api", "test-dependencies"] } -proptest = "1.0.0" -assert_matches = "1.3.0" -rand_xorshift = "0.3" -orchard = { version = "0.6", default-features = false, features = ["test-dependencies"] } +criterion.workspace = true +incrementalmerkletree = { workspace = true, features = ["legacy-api", "test-dependencies"] } +proptest.workspace = true +assert_matches.workspace = true +rand_xorshift.workspace = true +orchard = { workspace = true, features = ["test-dependencies"] } [target.'cfg(unix)'.dev-dependencies] pprof = { version = "0.11", features = ["criterion", "flamegraph"] } # MSRV 1.56 diff --git a/zcash_proofs/Cargo.toml b/zcash_proofs/Cargo.toml index a66295fd88..694c5f4482 100644 --- a/zcash_proofs/Cargo.toml +++ b/zcash_proofs/Cargo.toml @@ -6,33 +6,33 @@ authors = [ "Jack Grigg ", ] homepage = "https://github.com/zcash/librustzcash" -repository = "https://github.com/zcash/librustzcash" +repository.workspace = true readme = "README.md" -license = "MIT OR Apache-2.0" -edition = "2021" -rust-version = "1.65" -categories = ["cryptography::cryptocurrencies"] +license.workspace = true +edition.workspace = true +rust-version.workspace = true +categories.workspace = true [package.metadata.docs.rs] all-features = true [dependencies] -zcash_primitives = { version = "0.13", path = "../zcash_primitives", default-features = false } +zcash_primitives.workspace = true # Dependencies exposed in a public API: # (Breaking upgrades to these require a breaking upgrade to this crate.) bellman = { version = "0.14", default-features = false, features = ["groth16"] } -bls12_381 = "0.8" -group = "0.13" -jubjub = "0.10" -lazy_static = "1" +bls12_381.workspace = true +group.workspace = true +jubjub.workspace = true +lazy_static.workspace = true minreq = { version = "2", features = ["https"], optional = true } -rand_core = "0.6" -tracing = "0.1" +rand_core.workspace = true +tracing.workspace = true # Dependencies used internally: # (Breaking upgrades to these are usually backwards-compatible, but check MSRVs.) -blake2b_simd = "1" +blake2b_simd.workspace = true home = { version = "0.5", optional = true } known-folders = { version = "1", optional = true } redjubjub = "0.7" @@ -40,9 +40,9 @@ wagyu-zcash-parameters = { version = "0.2", optional = true } xdg = { version = "2.5", optional = true } [dev-dependencies] -byteorder = "1" -criterion = "0.4" -rand_xorshift = "0.3" +byteorder.workspace = true +criterion.workspace = true +rand_xorshift.workspace = true [target.'cfg(unix)'.dev-dependencies] pprof = { version = "0.11", features = ["criterion", "flamegraph"] } # MSRV 1.56 From 7b53b711e243e37d73e7e4821b3e7d5ac2858616 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 22 Sep 2023 21:43:43 +0000 Subject: [PATCH 0209/1122] CI: Fix Codecov job command `cargo tarpaulin` has changed the case of one of its config options. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d66d4c1372..eb9cd73f6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,7 +137,7 @@ jobs: run: cargo run --release --example download-params --features download-params - name: Generate coverage report - run: cargo tarpaulin --engine llvm --all-features --release --timeout 600 --out Xml + run: cargo tarpaulin --engine llvm --all-features --release --timeout 600 --out xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v3.1.4 From f6d7714efea9dc1d1bf51fd6a11834ecef5f45a5 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 27 Sep 2023 16:31:36 +0000 Subject: [PATCH 0210/1122] zcash_client_sqlite: Replace internal height tuples with `RangeInclusive` We don't need to iterate over them, but the `*_extrema` internal methods are semantically returning inclusive ranges, and using `RangeInclusive` avoids bugs where the wrong half of the tuple is used (instead moving the location of the tuple handling inside the `*_extrema` methods, which cuts the number of occurrences from linear in the number of function calls to constant). --- zcash_client_sqlite/src/lib.rs | 2 +- zcash_client_sqlite/src/wallet.rs | 27 ++++++++++--------- .../src/wallet/commitment_tree.rs | 2 +- .../init/migrations/shardtree_support.rs | 11 ++++---- zcash_client_sqlite/src/wallet/scanning.rs | 2 +- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 3ecf51ecac..34dcf5ff42 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -170,7 +170,7 @@ impl, P: consensus::Parameters> WalletRead for W fn chain_height(&self) -> Result, Self::Error> { wallet::scan_queue_extrema(self.conn.borrow()) - .map(|h| h.map(|(_, max)| max)) + .map(|h| h.map(|range| *range.end())) .map_err(SqliteClientError::from) } diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 70f52d97bf..715d30f7c5 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -72,6 +72,7 @@ use std::collections::{BTreeMap, HashMap}; use std::convert::TryFrom; use std::io::{self, Cursor}; use std::num::NonZeroU32; +use std::ops::RangeInclusive; use tracing::debug; use zcash_client_backend::data_api::{AccountBalance, Ratio, WalletSummary}; use zcash_primitives::transaction::components::amount::NonNegativeAmount; @@ -228,7 +229,7 @@ pub(crate) fn add_account( // Rewrite the scan ranges from the birthday height up to the chain tip so that we'll ensure we // re-scan to find any notes that might belong to the newly added account. - if let Some(t) = scan_queue_extrema(conn)?.map(|(_, max)| max) { + if let Some(t) = scan_queue_extrema(conn)?.map(|range| *range.end()) { let rescan_range = birthday.height()..(t + 1); replace_queue_entries::( @@ -592,7 +593,7 @@ pub(crate) fn get_wallet_summary( progress: &impl ScanProgress, ) -> Result, SqliteClientError> { let chain_tip_height = match scan_queue_extrema(conn)? { - Some((_, max)) => max, + Some(range) => *range.end(), None => { return Ok(None); } @@ -925,20 +926,20 @@ pub(crate) fn account_birthday( /// Returns the minimum and maximum heights for blocks stored in the wallet database. pub(crate) fn block_height_extrema( conn: &rusqlite::Connection, -) -> Result, rusqlite::Error> { +) -> Result>, rusqlite::Error> { conn.query_row("SELECT MIN(height), MAX(height) FROM blocks", [], |row| { let min_height: Option = row.get(0)?; let max_height: Option = row.get(1)?; Ok(min_height - .map(BlockHeight::from) - .zip(max_height.map(BlockHeight::from))) + .zip(max_height) + .map(|(min, max)| RangeInclusive::new(min.into(), max.into()))) }) } /// Returns the minimum and maximum heights of blocks in the chain which may be scanned. pub(crate) fn scan_queue_extrema( conn: &rusqlite::Connection, -) -> Result, rusqlite::Error> { +) -> Result>, rusqlite::Error> { conn.query_row( "SELECT MIN(block_range_start), MAX(block_range_end) FROM scan_queue", [], @@ -949,8 +950,8 @@ pub(crate) fn scan_queue_extrema( // Scan ranges are end-exclusive, so we subtract 1 from `max_height` to obtain the // height of the last known chain tip; Ok(min_height - .map(BlockHeight::from) - .zip(max_height.map(|h| BlockHeight::from(h.saturating_sub(1))))) + .zip(max_height.map(|h| h.saturating_sub(1))) + .map(|(min, max)| RangeInclusive::new(min.into(), max.into()))) }, ) } @@ -960,13 +961,13 @@ pub(crate) fn get_target_and_anchor_heights( min_confirmations: NonZeroU32, ) -> Result, rusqlite::Error> { scan_queue_extrema(conn).map(|heights| { - heights.map(|(min_height, max_height)| { - let target_height = max_height + 1; + heights.map(|range| { + let target_height = *range.end() + 1; // Select an anchor min_confirmations back from the target block, // unless that would be before the earliest block we have. let anchor_height = BlockHeight::from(cmp::max( u32::from(target_height).saturating_sub(min_confirmations.into()), - u32::from(min_height), + u32::from(*range.start()), )); (target_height, anchor_height) @@ -1306,7 +1307,7 @@ pub(crate) fn get_unspent_transparent_outputs( max_height: BlockHeight, exclude: &[OutPoint], ) -> Result, SqliteClientError> { - let chain_tip_height = scan_queue_extrema(conn)?.map(|(_, max)| max); + let chain_tip_height = scan_queue_extrema(conn)?.map(|range| *range.end()); let stable_height = chain_tip_height .unwrap_or(max_height) .saturating_sub(PRUNING_DEPTH); @@ -1377,7 +1378,7 @@ pub(crate) fn get_transparent_balances( account: AccountId, max_height: BlockHeight, ) -> Result, SqliteClientError> { - let chain_tip_height = scan_queue_extrema(conn)?.map(|(_, max)| max); + let chain_tip_height = scan_queue_extrema(conn)?.map(|range| *range.end()); let stable_height = chain_tip_height .unwrap_or(max_height) .saturating_sub(PRUNING_DEPTH); diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 81c74140c5..d0654063d0 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -756,7 +756,7 @@ pub(crate) fn get_checkpoint_depth( min_confirmations: NonZeroU32, ) -> Result, rusqlite::Error> { scan_queue_extrema(conn)? - .map(|(_, max)| max) + .map(|range| *range.end()) .map(|chain_tip| { let max_checkpoint_height = u32::from(chain_tip).saturating_sub(u32::from(min_confirmations) - 1); diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 822c87d9be..699deedc4d 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -157,13 +157,13 @@ impl RusqliteMigration for Migration

{ // `PRUNING_DEPTH`, and we won't be finding notes in earlier blocks), and // hurts performance (as frontier importing has a significant Merkle tree // hashing cost). - if let Some((nonempty_frontier, (_, latest_height))) = block_end_tree + if let Some((nonempty_frontier, scanned_range)) = block_end_tree .to_frontier() .value() - .zip(block_height_extrema) + .zip(block_height_extrema.as_ref()) { let block_height = BlockHeight::from(block_height); - if block_height + PRUNING_DEPTH >= latest_height { + if block_height + PRUNING_DEPTH >= *scanned_range.end() { trace!( height = u32::from(block_height), frontier = ?nonempty_frontier, @@ -256,9 +256,10 @@ impl RusqliteMigration for Migration

{ );", )?; - if let Some((start, end)) = block_height_extrema { + if let Some(scanned_range) = block_height_extrema { // `ScanRange` uses an exclusive upper bound. - let chain_end = end + 1; + let start = *scanned_range.start(); + let chain_end = *scanned_range.end() + 1; let ignored_range = self.params .activation_height(NetworkUpgrade::Sapling) diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index aa7050e699..41d1e922f0 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -346,7 +346,7 @@ pub(crate) fn update_chain_tip( }; // Read the previous max scanned height from the blocks table - let max_scanned = block_height_extrema(conn)?.map(|(_, max_scanned)| max_scanned); + let max_scanned = block_height_extrema(conn)?.map(|range| *range.end()); // Read the wallet birthday (if known). let wallet_birthday = wallet_birthday(conn)?; From a2e772c93ae4031ea8c1c972e76137b7aa6e6da6 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 27 Sep 2023 15:23:52 -0600 Subject: [PATCH 0211/1122] zcash_client_backend: Return summary information from `scan_cached_blocks` When scanning, a wallet only needs to update balance and transaction information shown to users when the scan has resulted in a change to wallet state. This modifies `scan_cached_blocks` to return the range of block heights actually scanned, along with the counts of notes spent and received by the wallet in that range. Fixes #918 --- zcash_client_backend/CHANGELOG.md | 6 +- zcash_client_backend/src/data_api/chain.rs | 92 +++++++++++++++++----- zcash_client_sqlite/src/chain.rs | 10 ++- zcash_client_sqlite/src/testing.rs | 13 ++- 4 files changed, 95 insertions(+), 26 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index aaa3bb93f6..449efeb5f3 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -7,9 +7,13 @@ and this library adheres to Rust's notion of ## [Unreleased] +### Changed +- `zcash_client_backend::data_api::chain::scan_cached_blocks` now returns + a `ScanSummary` containing metadata about the scanned blocks on success. + ## [0.10.0] - 2023-09-25 -### Notable Changes +### Notable Changes - `zcash_client_backend` now supports out-of-order scanning of blockchain history. See the module documentation for `zcash_client_backend::data_api::chain` for details on how to make use of the new scanning capabilities. diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 08ee524f72..e4353fc26a 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -144,6 +144,8 @@ //! # } //! ``` +use std::ops::Range; + use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight}, @@ -199,8 +201,6 @@ pub trait BlockSource { /// `from_height`, applying the provided callback to each block. If `from_height` /// is `None` then scanning will begin at the first available block. /// - /// Returns an error if `from_height` is set to a block that does not exist in `self`. - /// /// * `WalletErrT`: the types of errors produced by the wallet operations performed /// as part of processing each row. /// * `NoteRefT`: the type of note identifiers in the wallet data store, for use in @@ -209,29 +209,63 @@ pub trait BlockSource { &self, from_height: Option, limit: Option, - with_row: F, + with_block: F, ) -> Result<(), error::Error> where F: FnMut(CompactBlock) -> Result<(), error::Error>; } -/// Scans at most `limit` new blocks added to the block source for any transactions received by the -/// tracked accounts. -/// -/// If the `from_height` argument is not `None`, then this method block source will begin -/// requesting blocks from the provided block source at the specified height; if `from_height` is -/// `None then this will begin scanning at first block after the position to which the wallet has -/// previously fully scanned the chain, thereby beginning or continuing a linear scan over all +/// Metadata about modifications to the wallet state made in the course of scanning a set of /// blocks. +#[derive(Clone, Debug)] +pub struct ScanSummary { + scanned_range: Range, + spent_sapling_note_count: usize, + received_sapling_note_count: usize, +} + +impl ScanSummary { + /// Constructs a new [`ScanSummary`] from its constituent parts. + pub fn from_parts( + scanned_range: Range, + spent_sapling_note_count: usize, + received_sapling_note_count: usize, + ) -> Self { + Self { + scanned_range, + spent_sapling_note_count, + received_sapling_note_count, + } + } + + /// Returns the range of blocks successfully scanned. + pub fn scanned_range(&self) -> Range { + self.scanned_range.clone() + } + + /// Returns the number of our previously-detected Sapling notes that were spent in transactions + /// in blocks in the scanned range. If we have not yet detected a particular note as ours, for + /// example because we are scanning the chain in reverse height order, we will not detect it + /// being spent at this time. + pub fn spent_sapling_note_count(&self) -> usize { + self.spent_sapling_note_count + } + + /// Returns the number of notes belonging to the wallet that were received in blocks in the + /// scanned range. Note that depending upon the scanning order, it is possible that some of the + /// received notes counted here may already have been spent in later blocks closer to the chain + /// tip. + pub fn received_sapling_note_count(&self) -> usize { + self.received_sapling_note_count + } +} + +/// Scans at most `limit` blocks from the provided block source for in order to find transactions +/// received by the accounts tracked in the provided wallet database. /// -/// This function will return without error after scanning at most `limit` new blocks, to enable -/// the caller to update their UI with scanning progress. Repeatedly calling this function with -/// `from_height == None` will process sequential ranges of blocks. -/// -/// For brand-new light client databases, if `from_height == None` this function starts scanning -/// from the Sapling activation height. This height can be fast-forwarded to a more recent block by -/// initializing the client database with a starting block (for example, calling -/// `init_blocks_table` before this function if using `zcash_client_sqlite`). +/// This function will return after scanning at most `limit` new blocks, to enable the caller to +/// update their UI with scanning progress. Repeatedly calling this function with `from_height == +/// None` will process sequential ranges of blocks. #[tracing::instrument(skip(params, block_source, data_db))] #[allow(clippy::type_complexity)] pub fn scan_cached_blocks( @@ -240,7 +274,7 @@ pub fn scan_cached_blocks( data_db: &mut DbT, from_height: BlockHeight, limit: usize, -) -> Result<(), Error> +) -> Result> where ParamsT: consensus::Parameters + Send + 'static, BlockSourceT: BlockSource, @@ -341,10 +375,14 @@ where batch_runner.flush(); let mut scanned_blocks = vec![]; + let mut scan_end_height = from_height; + let mut received_note_count = 0; + let mut spent_note_count = 0; block_source.with_blocks::<_, DbT::Error>( Some(from_height), Some(limit), |block: CompactBlock| { + scan_end_height = block.height() + 1; let scanned_block = scan_block_with_runner( params, block, @@ -355,6 +393,15 @@ where ) .map_err(Error::Scan)?; + let (s, r) = scanned_block + .transactions + .iter() + .fold((0, 0), |(s, r), wtx| { + (s + wtx.sapling_spends.len(), r + wtx.sapling_outputs.len()) + }); + spent_note_count += s; + received_note_count += r; + let spent_nf: Vec<&sapling::Nullifier> = scanned_block .transactions .iter() @@ -370,12 +417,17 @@ where prior_block_metadata = Some(*scanned_block.metadata()); scanned_blocks.push(scanned_block); + Ok(()) }, )?; data_db.put_blocks(scanned_blocks).map_err(Error::Wallet)?; - Ok(()) + Ok(ScanSummary::from_parts( + from_height..scan_end_height, + spent_note_count, + received_note_count, + )) } #[cfg(feature = "test-dependencies")] diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 950d2e2b30..b7a7c63e04 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -564,7 +564,10 @@ mod tests { let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); // Scan the cache - st.scan_cached_blocks(h1, 1); + let summary = st.scan_cached_blocks(h1, 1); + assert_eq!(summary.scanned_range().start, h1); + assert_eq!(summary.scanned_range().end, h1 + 1); + assert_eq!(summary.received_sapling_note_count(), 1); // Account balance should reflect the received note assert_eq!(st.get_total_balance(AccountId::from(0)), value); @@ -574,7 +577,10 @@ mod tests { let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2.into()); // Scan the cache again - st.scan_cached_blocks(h2, 1); + let summary = st.scan_cached_blocks(h2, 1); + assert_eq!(summary.scanned_range().start, h2); + assert_eq!(summary.scanned_range().end, h2 + 1); + assert_eq!(summary.received_sapling_note_count(), 1); // Account balance should reflect both received notes assert_eq!( diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 5bc160ba09..386543c10c 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -14,6 +14,7 @@ use tempfile::NamedTempFile; #[cfg(feature = "unstable")] use tempfile::TempDir; +use zcash_client_backend::data_api::chain::ScanSummary; use zcash_client_backend::data_api::{AccountBalance, WalletRead}; #[allow(deprecated)] use zcash_client_backend::{ @@ -326,8 +327,14 @@ where } /// Invokes [`scan_cached_blocks`] with the given arguments, expecting success. - pub(crate) fn scan_cached_blocks(&mut self, from_height: BlockHeight, limit: usize) { - assert_matches!(self.try_scan_cached_blocks(from_height, limit), Ok(_)); + pub(crate) fn scan_cached_blocks( + &mut self, + from_height: BlockHeight, + limit: usize, + ) -> ScanSummary { + let result = self.try_scan_cached_blocks(from_height, limit); + assert_matches!(result, Ok(_)); + result.unwrap() } /// Invokes [`scan_cached_blocks`] with the given arguments. @@ -336,7 +343,7 @@ where from_height: BlockHeight, limit: usize, ) -> Result< - (), + ScanSummary, data_api::chain::error::Error< SqliteClientError, ::Error, From c16e77722258017fa2fb1f8d1fa1d5d51720dc33 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 28 Sep 2023 16:40:03 -0600 Subject: [PATCH 0212/1122] zcash_client_sqlite: Fix zcash_proofs dev-dependency --- zcash_client_sqlite/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index a0a9009764..8a3a52a3d9 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -61,7 +61,7 @@ rand_core.workspace = true regex = "1.4" tempfile = "3.5.0" zcash_note_encryption.workspace = true -zcash_proofs.workspace = true +zcash_proofs = { workspace = true, features = ["local-prover"] } zcash_primitives = { workspace = true, features = ["test-dependencies"] } zcash_client_backend = { workspace = true, features = ["test-dependencies", "unstable-serialization", "unstable-spanning-tree"] } zcash_address = { workspace = true, features = ["test-dependencies"] } From 0ddcccdbac37c9e48e1171b06e369be6edb11aea Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 29 Sep 2023 16:52:09 +0000 Subject: [PATCH 0213/1122] zcash_primitives: Move Sapling constants into `sapling` module --- zcash_client_backend/src/scanning.rs | 2 +- zcash_primitives/CHANGELOG.md | 7 + zcash_primitives/src/constants.rs | 435 ----------------- zcash_primitives/src/sapling.rs | 3 +- zcash_primitives/src/sapling/constants.rs | 436 ++++++++++++++++++ zcash_primitives/src/sapling/group_hash.rs | 2 +- zcash_primitives/src/sapling/keys.rs | 8 +- zcash_primitives/src/sapling/pedersen_hash.rs | 2 +- zcash_primitives/src/sapling/prover.rs | 2 +- zcash_primitives/src/sapling/redjubjub.rs | 2 +- zcash_primitives/src/sapling/spec.rs | 8 +- zcash_primitives/src/sapling/value.rs | 2 +- zcash_primitives/src/sapling/value/sums.rs | 3 +- .../src/transaction/components/sapling.rs | 2 +- zcash_primitives/src/zip32/sapling.rs | 8 +- zcash_proofs/src/circuit/ecc.rs | 2 +- zcash_proofs/src/circuit/sapling.rs | 4 +- zcash_proofs/src/constants.rs | 16 +- zcash_proofs/src/sapling/prover.rs | 2 +- zcash_proofs/src/sapling/verifier/single.rs | 2 +- 20 files changed, 478 insertions(+), 470 deletions(-) create mode 100644 zcash_primitives/src/sapling/constants.rs diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index ba72591844..919946040d 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -571,10 +571,10 @@ mod tests { use zcash_primitives::{ block::BlockHash, consensus::{BlockHeight, Network}, - constants::SPENDING_KEY_GENERATOR, memo::MemoBytes, sapling::{ self, + constants::SPENDING_KEY_GENERATOR, note_encryption::{sapling_note_encryption, PreparedIncomingViewingKey, SaplingDomain}, util::generate_random_rseed, value::NoteValue, diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index b0450f2102..f7f326728b 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -6,6 +6,13 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- `zcash_primitives::sapling`: + - `constants` module. + +### Removed +- `zcash_primitives::constants`: + - All `const` values (moved to `zcash_primitives::sapling::constants`). ## [0.13.0] - 2023-09-25 ### Added diff --git a/zcash_primitives/src/constants.rs b/zcash_primitives/src/constants.rs index 95bcd53262..2643d912b5 100644 --- a/zcash_primitives/src/constants.rs +++ b/zcash_primitives/src/constants.rs @@ -1,440 +1,5 @@ //! Various constants used by the Zcash primitives. -use ff::PrimeField; -use group::Group; -use jubjub::SubgroupPoint; -use lazy_static::lazy_static; - pub mod mainnet; pub mod regtest; pub mod testnet; - -/// First 64 bytes of the BLAKE2s input during group hash. -/// This is chosen to be some random string that we couldn't have anticipated when we designed -/// the algorithm, for rigidity purposes. -/// We deliberately use an ASCII hex string of 32 bytes here. -pub const GH_FIRST_BLOCK: &[u8; 64] = - b"096b36a5804bfacef1691e173c366a47ff5ba84a44f26ddd7e8d9f79d5b42df0"; - -// BLAKE2s invocation personalizations -/// BLAKE2s Personalization for CRH^ivk = BLAKE2s(ak | nk) -pub const CRH_IVK_PERSONALIZATION: &[u8; 8] = b"Zcashivk"; - -/// BLAKE2s Personalization for PRF^nf = BLAKE2s(nk | rho) -pub const PRF_NF_PERSONALIZATION: &[u8; 8] = b"Zcash_nf"; - -// Group hash personalizations -/// BLAKE2s Personalization for Pedersen hash generators. -pub const PEDERSEN_HASH_GENERATORS_PERSONALIZATION: &[u8; 8] = b"Zcash_PH"; - -/// BLAKE2s Personalization for the group hash for key diversification -pub const KEY_DIVERSIFICATION_PERSONALIZATION: &[u8; 8] = b"Zcash_gd"; - -/// BLAKE2s Personalization for the spending key base point -pub const SPENDING_KEY_GENERATOR_PERSONALIZATION: &[u8; 8] = b"Zcash_G_"; - -/// BLAKE2s Personalization for the proof generation key base point -pub const PROOF_GENERATION_KEY_BASE_GENERATOR_PERSONALIZATION: &[u8; 8] = b"Zcash_H_"; - -/// BLAKE2s Personalization for the value commitment generator for the value -pub const VALUE_COMMITMENT_GENERATOR_PERSONALIZATION: &[u8; 8] = b"Zcash_cv"; - -/// BLAKE2s Personalization for the nullifier position generator (for computing rho) -pub const NULLIFIER_POSITION_IN_TREE_GENERATOR_PERSONALIZATION: &[u8; 8] = b"Zcash_J_"; - -/// The prover will demonstrate knowledge of discrete log with respect to this base when -/// they are constructing a proof, in order to authorize proof construction. -pub const PROOF_GENERATION_KEY_GENERATOR: SubgroupPoint = SubgroupPoint::from_raw_unchecked( - bls12_381::Scalar::from_raw([ - 0x3af2_dbef_b96e_2571, - 0xadf2_d038_f2fb_b820, - 0x7043_03f1_e890_6081, - 0x1457_a502_31cd_e2df, - ]), - bls12_381::Scalar::from_raw([ - 0x467a_f9f7_e05d_e8e7, - 0x50df_51ea_f5a1_49d2, - 0xdec9_0184_0f49_48cc, - 0x54b6_d107_18df_2a7a, - ]), -); - -/// The note commitment is randomized over this generator. -pub const NOTE_COMMITMENT_RANDOMNESS_GENERATOR: SubgroupPoint = SubgroupPoint::from_raw_unchecked( - bls12_381::Scalar::from_raw([ - 0xa514_3b34_a8e3_6462, - 0xf091_9d06_ffb1_ecda, - 0xa140_9aa1_f33b_ec2c, - 0x26eb_9f8a_9ec7_2a8c, - ]), - bls12_381::Scalar::from_raw([ - 0xd4fc_6365_796c_77ac, - 0x96b7_8bea_fa9c_c44c, - 0x949d_7747_6e26_2c95, - 0x114b_7501_ad10_4c57, - ]), -); - -/// The node commitment is randomized again by the position in order to supply the -/// nullifier computation with a unique input w.r.t. the note being spent, to prevent -/// Faerie gold attacks. -pub const NULLIFIER_POSITION_GENERATOR: SubgroupPoint = SubgroupPoint::from_raw_unchecked( - bls12_381::Scalar::from_raw([ - 0x2ce3_3921_888d_30db, - 0xe81c_ee09_a561_229e, - 0xdb56_b6db_8d80_75ed, - 0x2400_c2e2_e336_2644, - ]), - bls12_381::Scalar::from_raw([ - 0xa3f7_fa36_c72b_0065, - 0xe155_b8e8_ffff_2e42, - 0xfc9e_8a15_a096_ba8f, - 0x6136_9d54_40bf_84a5, - ]), -); - -/// The value commitment is used to check balance between inputs and outputs. The value is -/// placed over this generator. -pub const VALUE_COMMITMENT_VALUE_GENERATOR: SubgroupPoint = SubgroupPoint::from_raw_unchecked( - bls12_381::Scalar::from_raw([ - 0x3618_3b2c_b4d7_ef51, - 0x9472_c89a_c043_042d, - 0xd861_8ed1_d15f_ef4e, - 0x273f_910d_9ecc_1615, - ]), - bls12_381::Scalar::from_raw([ - 0xa77a_81f5_0667_c8d7, - 0xbc33_32d0_fa1c_cd18, - 0xd322_94fd_8977_4ad6, - 0x466a_7e3a_82f6_7ab1, - ]), -); - -/// The value commitment is randomized over this generator, for privacy. -pub const VALUE_COMMITMENT_RANDOMNESS_GENERATOR: SubgroupPoint = SubgroupPoint::from_raw_unchecked( - bls12_381::Scalar::from_raw([ - 0x3bce_3b77_9366_4337, - 0xd1d8_da41_af03_744e, - 0x7ff6_826a_d580_04b4, - 0x6800_f4fa_0f00_1cfc, - ]), - bls12_381::Scalar::from_raw([ - 0x3cae_fab9_380b_6a8b, - 0xad46_f1b0_473b_803b, - 0xe6fb_2a6e_1e22_ab50, - 0x6d81_d3a9_cb45_dedb, - ]), -); - -/// The spender proves discrete log with respect to this base at spend time. -pub const SPENDING_KEY_GENERATOR: SubgroupPoint = SubgroupPoint::from_raw_unchecked( - bls12_381::Scalar::from_raw([ - 0x47bf_4692_0a95_a753, - 0xd5b9_a7d3_ef8e_2827, - 0xd418_a7ff_2675_3b6a, - 0x0926_d4f3_2059_c712, - ]), - bls12_381::Scalar::from_raw([ - 0x3056_32ad_aaf2_b530, - 0x6d65_674d_cedb_ddbc, - 0x53bb_37d0_c21c_fd05, - 0x57a1_019e_6de9_b675, - ]), -); - -/// The generators (for each segment) used in all Pedersen commitments. -pub const PEDERSEN_HASH_GENERATORS: &[SubgroupPoint] = &[ - SubgroupPoint::from_raw_unchecked( - bls12_381::Scalar::from_raw([ - 0x194e_4292_6f66_1b51, - 0x2f0c_718f_6f0f_badd, - 0xb5ea_25de_7ec0_e378, - 0x73c0_16a4_2ded_9578, - ]), - bls12_381::Scalar::from_raw([ - 0x77bf_abd4_3224_3cca, - 0xf947_2e8b_c04e_4632, - 0x79c9_166b_837e_dc5e, - 0x289e_87a2_d352_1b57, - ]), - ), - SubgroupPoint::from_raw_unchecked( - bls12_381::Scalar::from_raw([ - 0xb981_9dc8_2d90_607e, - 0xa361_ee3f_d48f_df77, - 0x52a3_5a8c_1908_dd87, - 0x15a3_6d1f_0f39_0d88, - ]), - bls12_381::Scalar::from_raw([ - 0x7b0d_c53c_4ebf_1891, - 0x1f3a_beeb_98fa_d3e8, - 0xf789_1142_c001_d925, - 0x015d_8c7f_5b43_fe33, - ]), - ), - SubgroupPoint::from_raw_unchecked( - bls12_381::Scalar::from_raw([ - 0x76d6_f7c2_b67f_c475, - 0xbae8_e5c4_6641_ae5c, - 0xeb69_ae39_f5c8_4210, - 0x6643_21a5_8246_e2f6, - ]), - bls12_381::Scalar::from_raw([ - 0x80ed_502c_9793_d457, - 0x8bb2_2a7f_1784_b498, - 0xe000_a46c_8e8c_e853, - 0x362e_1500_d24e_ee9e, - ]), - ), - SubgroupPoint::from_raw_unchecked( - bls12_381::Scalar::from_raw([ - 0x4c76_7804_c1c4_a2cc, - 0x7d02_d50e_654b_87f2, - 0xedc5_f4a9_cff2_9fd5, - 0x323a_6548_ce9d_9876, - ]), - bls12_381::Scalar::from_raw([ - 0x8471_4bec_a335_70e9, - 0x5103_afa1_a11f_6a85, - 0x9107_0acb_d8d9_47b7, - 0x2f7e_e40c_4b56_cad8, - ]), - ), - SubgroupPoint::from_raw_unchecked( - bls12_381::Scalar::from_raw([ - 0x4680_9430_657f_82d1, - 0xefd5_9313_05f2_f0bf, - 0x89b6_4b4e_0336_2796, - 0x3bd2_6660_00b5_4796, - ]), - bls12_381::Scalar::from_raw([ - 0x9996_8299_c365_8aef, - 0xb3b9_d809_5859_d14c, - 0x3978_3238_1406_c9e5, - 0x494b_c521_03ab_9d0a, - ]), - ), - SubgroupPoint::from_raw_unchecked( - bls12_381::Scalar::from_raw([ - 0xcb3c_0232_58d3_2079, - 0x1d9e_5ca2_1135_ff6f, - 0xda04_9746_d76d_3ee5, - 0x6344_7b2b_a31b_b28a, - ]), - bls12_381::Scalar::from_raw([ - 0x4360_8211_9f8d_629a, - 0xa802_00d2_c66b_13a7, - 0x64cd_b107_0a13_6a28, - 0x64ec_4689_e8bf_b6e5, - ]), - ), -]; - -/// The maximum number of chunks per segment of the Pedersen hash. -pub const PEDERSEN_HASH_CHUNKS_PER_GENERATOR: usize = 63; - -/// The window size for exponentiation of Pedersen hash generators outside the circuit. -pub const PEDERSEN_HASH_EXP_WINDOW_SIZE: u32 = 8; - -lazy_static! { - /// The exp table for [`PEDERSEN_HASH_GENERATORS`]. - pub static ref PEDERSEN_HASH_EXP_TABLE: Vec>> = - generate_pedersen_hash_exp_table(); -} - -/// Creates the exp table for the Pedersen hash generators. -fn generate_pedersen_hash_exp_table() -> Vec>> { - let window = PEDERSEN_HASH_EXP_WINDOW_SIZE; - - PEDERSEN_HASH_GENERATORS - .iter() - .cloned() - .map(|mut g| { - let mut tables = vec![]; - - let mut num_bits = 0; - while num_bits <= jubjub::Fr::NUM_BITS { - let mut table = Vec::with_capacity(1 << window); - let mut base = SubgroupPoint::identity(); - - for _ in 0..(1 << window) { - table.push(base); - base += g; - } - - tables.push(table); - num_bits += window; - - for _ in 0..window { - g = g.double(); - } - } - - tables - }) - .collect() -} - -#[cfg(test)] -mod tests { - use jubjub::SubgroupPoint; - - use super::*; - use crate::sapling::group_hash::group_hash; - - fn find_group_hash(m: &[u8], personalization: &[u8; 8]) -> SubgroupPoint { - let mut tag = m.to_vec(); - let i = tag.len(); - tag.push(0u8); - - loop { - let gh = group_hash(&tag, personalization); - - // We don't want to overflow and start reusing generators - assert!(tag[i] != u8::max_value()); - tag[i] += 1; - - if let Some(gh) = gh { - break gh; - } - } - } - - #[test] - fn proof_generation_key_base_generator() { - assert_eq!( - find_group_hash(&[], PROOF_GENERATION_KEY_BASE_GENERATOR_PERSONALIZATION), - PROOF_GENERATION_KEY_GENERATOR, - ); - } - - #[test] - fn note_commitment_randomness_generator() { - assert_eq!( - find_group_hash(b"r", PEDERSEN_HASH_GENERATORS_PERSONALIZATION), - NOTE_COMMITMENT_RANDOMNESS_GENERATOR, - ); - } - - #[test] - fn nullifier_position_generator() { - assert_eq!( - find_group_hash(&[], NULLIFIER_POSITION_IN_TREE_GENERATOR_PERSONALIZATION), - NULLIFIER_POSITION_GENERATOR, - ); - } - - #[test] - fn value_commitment_value_generator() { - assert_eq!( - find_group_hash(b"v", VALUE_COMMITMENT_GENERATOR_PERSONALIZATION), - VALUE_COMMITMENT_VALUE_GENERATOR, - ); - } - - #[test] - fn value_commitment_randomness_generator() { - assert_eq!( - find_group_hash(b"r", VALUE_COMMITMENT_GENERATOR_PERSONALIZATION), - VALUE_COMMITMENT_RANDOMNESS_GENERATOR, - ); - } - - #[test] - fn spending_key_generator() { - assert_eq!( - find_group_hash(&[], SPENDING_KEY_GENERATOR_PERSONALIZATION), - SPENDING_KEY_GENERATOR, - ); - } - - #[test] - fn pedersen_hash_generators() { - for (m, actual) in PEDERSEN_HASH_GENERATORS.iter().enumerate() { - assert_eq!( - &find_group_hash( - &(m as u32).to_le_bytes(), - PEDERSEN_HASH_GENERATORS_PERSONALIZATION - ), - actual - ); - } - } - - #[test] - fn no_duplicate_fixed_base_generators() { - let fixed_base_generators = [ - PROOF_GENERATION_KEY_GENERATOR, - NOTE_COMMITMENT_RANDOMNESS_GENERATOR, - NULLIFIER_POSITION_GENERATOR, - VALUE_COMMITMENT_VALUE_GENERATOR, - VALUE_COMMITMENT_RANDOMNESS_GENERATOR, - SPENDING_KEY_GENERATOR, - ]; - - // Check for duplicates, far worse than spec inconsistencies! - for (i, p1) in fixed_base_generators.iter().enumerate() { - if p1.is_identity().into() { - panic!("Neutral element!"); - } - - for p2 in fixed_base_generators.iter().skip(i + 1) { - if p1 == p2 { - panic!("Duplicate generator!"); - } - } - } - } - - /// Check for simple relations between the generators, that make finding collisions easy; - /// far worse than spec inconsistencies! - fn check_consistency_of_pedersen_hash_generators( - pedersen_hash_generators: &[jubjub::SubgroupPoint], - ) { - for (i, p1) in pedersen_hash_generators.iter().enumerate() { - if p1.is_identity().into() { - panic!("Neutral element!"); - } - for p2 in pedersen_hash_generators.iter().skip(i + 1) { - if p1 == p2 { - panic!("Duplicate generator!"); - } - if *p1 == -p2 { - panic!("Inverse generator!"); - } - } - - // check for a generator being the sum of any other two - for (j, p2) in pedersen_hash_generators.iter().enumerate() { - if j == i { - continue; - } - for (k, p3) in pedersen_hash_generators.iter().enumerate() { - if k == j || k == i { - continue; - } - let sum = p2 + p3; - if sum == *p1 { - panic!("Linear relation between generators!"); - } - } - } - } - } - - #[test] - fn pedersen_hash_generators_consistency() { - check_consistency_of_pedersen_hash_generators(PEDERSEN_HASH_GENERATORS); - } - - #[test] - #[should_panic(expected = "Linear relation between generators!")] - fn test_jubjub_bls12_pedersen_hash_generators_consistency_check_linear_relation() { - let mut pedersen_hash_generators = PEDERSEN_HASH_GENERATORS.to_vec(); - - // Test for linear relation - pedersen_hash_generators.push(PEDERSEN_HASH_GENERATORS[0] + PEDERSEN_HASH_GENERATORS[1]); - - check_consistency_of_pedersen_hash_generators(&pedersen_hash_generators); - } -} diff --git a/zcash_primitives/src/sapling.rs b/zcash_primitives/src/sapling.rs index 31899c72be..7cccd252f7 100644 --- a/zcash_primitives/src/sapling.rs +++ b/zcash_primitives/src/sapling.rs @@ -1,6 +1,7 @@ //! Structs and constants specific to the Sapling shielded pool. mod address; +pub mod constants; pub mod group_hash; pub mod keys; pub mod note; @@ -16,7 +17,7 @@ pub mod value; use group::GroupEncoding; use rand_core::{CryptoRng, RngCore}; -use crate::constants::SPENDING_KEY_GENERATOR; +use constants::SPENDING_KEY_GENERATOR; use self::redjubjub::{PrivateKey, PublicKey, Signature}; diff --git a/zcash_primitives/src/sapling/constants.rs b/zcash_primitives/src/sapling/constants.rs new file mode 100644 index 0000000000..e9c4e43996 --- /dev/null +++ b/zcash_primitives/src/sapling/constants.rs @@ -0,0 +1,436 @@ +//! Various constants used by the Sapling protocol. + +use ff::PrimeField; +use group::Group; +use jubjub::SubgroupPoint; +use lazy_static::lazy_static; + +/// First 64 bytes of the BLAKE2s input during group hash. +/// This is chosen to be some random string that we couldn't have anticipated when we designed +/// the algorithm, for rigidity purposes. +/// We deliberately use an ASCII hex string of 32 bytes here. +pub const GH_FIRST_BLOCK: &[u8; 64] = + b"096b36a5804bfacef1691e173c366a47ff5ba84a44f26ddd7e8d9f79d5b42df0"; + +// BLAKE2s invocation personalizations +/// BLAKE2s Personalization for CRH^ivk = BLAKE2s(ak | nk) +pub const CRH_IVK_PERSONALIZATION: &[u8; 8] = b"Zcashivk"; + +/// BLAKE2s Personalization for PRF^nf = BLAKE2s(nk | rho) +pub const PRF_NF_PERSONALIZATION: &[u8; 8] = b"Zcash_nf"; + +// Group hash personalizations +/// BLAKE2s Personalization for Pedersen hash generators. +pub const PEDERSEN_HASH_GENERATORS_PERSONALIZATION: &[u8; 8] = b"Zcash_PH"; + +/// BLAKE2s Personalization for the group hash for key diversification +pub const KEY_DIVERSIFICATION_PERSONALIZATION: &[u8; 8] = b"Zcash_gd"; + +/// BLAKE2s Personalization for the spending key base point +pub const SPENDING_KEY_GENERATOR_PERSONALIZATION: &[u8; 8] = b"Zcash_G_"; + +/// BLAKE2s Personalization for the proof generation key base point +pub const PROOF_GENERATION_KEY_BASE_GENERATOR_PERSONALIZATION: &[u8; 8] = b"Zcash_H_"; + +/// BLAKE2s Personalization for the value commitment generator for the value +pub const VALUE_COMMITMENT_GENERATOR_PERSONALIZATION: &[u8; 8] = b"Zcash_cv"; + +/// BLAKE2s Personalization for the nullifier position generator (for computing rho) +pub const NULLIFIER_POSITION_IN_TREE_GENERATOR_PERSONALIZATION: &[u8; 8] = b"Zcash_J_"; + +/// The prover will demonstrate knowledge of discrete log with respect to this base when +/// they are constructing a proof, in order to authorize proof construction. +pub const PROOF_GENERATION_KEY_GENERATOR: SubgroupPoint = SubgroupPoint::from_raw_unchecked( + bls12_381::Scalar::from_raw([ + 0x3af2_dbef_b96e_2571, + 0xadf2_d038_f2fb_b820, + 0x7043_03f1_e890_6081, + 0x1457_a502_31cd_e2df, + ]), + bls12_381::Scalar::from_raw([ + 0x467a_f9f7_e05d_e8e7, + 0x50df_51ea_f5a1_49d2, + 0xdec9_0184_0f49_48cc, + 0x54b6_d107_18df_2a7a, + ]), +); + +/// The note commitment is randomized over this generator. +pub const NOTE_COMMITMENT_RANDOMNESS_GENERATOR: SubgroupPoint = SubgroupPoint::from_raw_unchecked( + bls12_381::Scalar::from_raw([ + 0xa514_3b34_a8e3_6462, + 0xf091_9d06_ffb1_ecda, + 0xa140_9aa1_f33b_ec2c, + 0x26eb_9f8a_9ec7_2a8c, + ]), + bls12_381::Scalar::from_raw([ + 0xd4fc_6365_796c_77ac, + 0x96b7_8bea_fa9c_c44c, + 0x949d_7747_6e26_2c95, + 0x114b_7501_ad10_4c57, + ]), +); + +/// The node commitment is randomized again by the position in order to supply the +/// nullifier computation with a unique input w.r.t. the note being spent, to prevent +/// Faerie gold attacks. +pub const NULLIFIER_POSITION_GENERATOR: SubgroupPoint = SubgroupPoint::from_raw_unchecked( + bls12_381::Scalar::from_raw([ + 0x2ce3_3921_888d_30db, + 0xe81c_ee09_a561_229e, + 0xdb56_b6db_8d80_75ed, + 0x2400_c2e2_e336_2644, + ]), + bls12_381::Scalar::from_raw([ + 0xa3f7_fa36_c72b_0065, + 0xe155_b8e8_ffff_2e42, + 0xfc9e_8a15_a096_ba8f, + 0x6136_9d54_40bf_84a5, + ]), +); + +/// The value commitment is used to check balance between inputs and outputs. The value is +/// placed over this generator. +pub const VALUE_COMMITMENT_VALUE_GENERATOR: SubgroupPoint = SubgroupPoint::from_raw_unchecked( + bls12_381::Scalar::from_raw([ + 0x3618_3b2c_b4d7_ef51, + 0x9472_c89a_c043_042d, + 0xd861_8ed1_d15f_ef4e, + 0x273f_910d_9ecc_1615, + ]), + bls12_381::Scalar::from_raw([ + 0xa77a_81f5_0667_c8d7, + 0xbc33_32d0_fa1c_cd18, + 0xd322_94fd_8977_4ad6, + 0x466a_7e3a_82f6_7ab1, + ]), +); + +/// The value commitment is randomized over this generator, for privacy. +pub const VALUE_COMMITMENT_RANDOMNESS_GENERATOR: SubgroupPoint = SubgroupPoint::from_raw_unchecked( + bls12_381::Scalar::from_raw([ + 0x3bce_3b77_9366_4337, + 0xd1d8_da41_af03_744e, + 0x7ff6_826a_d580_04b4, + 0x6800_f4fa_0f00_1cfc, + ]), + bls12_381::Scalar::from_raw([ + 0x3cae_fab9_380b_6a8b, + 0xad46_f1b0_473b_803b, + 0xe6fb_2a6e_1e22_ab50, + 0x6d81_d3a9_cb45_dedb, + ]), +); + +/// The spender proves discrete log with respect to this base at spend time. +pub const SPENDING_KEY_GENERATOR: SubgroupPoint = SubgroupPoint::from_raw_unchecked( + bls12_381::Scalar::from_raw([ + 0x47bf_4692_0a95_a753, + 0xd5b9_a7d3_ef8e_2827, + 0xd418_a7ff_2675_3b6a, + 0x0926_d4f3_2059_c712, + ]), + bls12_381::Scalar::from_raw([ + 0x3056_32ad_aaf2_b530, + 0x6d65_674d_cedb_ddbc, + 0x53bb_37d0_c21c_fd05, + 0x57a1_019e_6de9_b675, + ]), +); + +/// The generators (for each segment) used in all Pedersen commitments. +pub const PEDERSEN_HASH_GENERATORS: &[SubgroupPoint] = &[ + SubgroupPoint::from_raw_unchecked( + bls12_381::Scalar::from_raw([ + 0x194e_4292_6f66_1b51, + 0x2f0c_718f_6f0f_badd, + 0xb5ea_25de_7ec0_e378, + 0x73c0_16a4_2ded_9578, + ]), + bls12_381::Scalar::from_raw([ + 0x77bf_abd4_3224_3cca, + 0xf947_2e8b_c04e_4632, + 0x79c9_166b_837e_dc5e, + 0x289e_87a2_d352_1b57, + ]), + ), + SubgroupPoint::from_raw_unchecked( + bls12_381::Scalar::from_raw([ + 0xb981_9dc8_2d90_607e, + 0xa361_ee3f_d48f_df77, + 0x52a3_5a8c_1908_dd87, + 0x15a3_6d1f_0f39_0d88, + ]), + bls12_381::Scalar::from_raw([ + 0x7b0d_c53c_4ebf_1891, + 0x1f3a_beeb_98fa_d3e8, + 0xf789_1142_c001_d925, + 0x015d_8c7f_5b43_fe33, + ]), + ), + SubgroupPoint::from_raw_unchecked( + bls12_381::Scalar::from_raw([ + 0x76d6_f7c2_b67f_c475, + 0xbae8_e5c4_6641_ae5c, + 0xeb69_ae39_f5c8_4210, + 0x6643_21a5_8246_e2f6, + ]), + bls12_381::Scalar::from_raw([ + 0x80ed_502c_9793_d457, + 0x8bb2_2a7f_1784_b498, + 0xe000_a46c_8e8c_e853, + 0x362e_1500_d24e_ee9e, + ]), + ), + SubgroupPoint::from_raw_unchecked( + bls12_381::Scalar::from_raw([ + 0x4c76_7804_c1c4_a2cc, + 0x7d02_d50e_654b_87f2, + 0xedc5_f4a9_cff2_9fd5, + 0x323a_6548_ce9d_9876, + ]), + bls12_381::Scalar::from_raw([ + 0x8471_4bec_a335_70e9, + 0x5103_afa1_a11f_6a85, + 0x9107_0acb_d8d9_47b7, + 0x2f7e_e40c_4b56_cad8, + ]), + ), + SubgroupPoint::from_raw_unchecked( + bls12_381::Scalar::from_raw([ + 0x4680_9430_657f_82d1, + 0xefd5_9313_05f2_f0bf, + 0x89b6_4b4e_0336_2796, + 0x3bd2_6660_00b5_4796, + ]), + bls12_381::Scalar::from_raw([ + 0x9996_8299_c365_8aef, + 0xb3b9_d809_5859_d14c, + 0x3978_3238_1406_c9e5, + 0x494b_c521_03ab_9d0a, + ]), + ), + SubgroupPoint::from_raw_unchecked( + bls12_381::Scalar::from_raw([ + 0xcb3c_0232_58d3_2079, + 0x1d9e_5ca2_1135_ff6f, + 0xda04_9746_d76d_3ee5, + 0x6344_7b2b_a31b_b28a, + ]), + bls12_381::Scalar::from_raw([ + 0x4360_8211_9f8d_629a, + 0xa802_00d2_c66b_13a7, + 0x64cd_b107_0a13_6a28, + 0x64ec_4689_e8bf_b6e5, + ]), + ), +]; + +/// The maximum number of chunks per segment of the Pedersen hash. +pub const PEDERSEN_HASH_CHUNKS_PER_GENERATOR: usize = 63; + +/// The window size for exponentiation of Pedersen hash generators outside the circuit. +pub const PEDERSEN_HASH_EXP_WINDOW_SIZE: u32 = 8; + +lazy_static! { + /// The exp table for [`PEDERSEN_HASH_GENERATORS`]. + pub static ref PEDERSEN_HASH_EXP_TABLE: Vec>> = + generate_pedersen_hash_exp_table(); +} + +/// Creates the exp table for the Pedersen hash generators. +fn generate_pedersen_hash_exp_table() -> Vec>> { + let window = PEDERSEN_HASH_EXP_WINDOW_SIZE; + + PEDERSEN_HASH_GENERATORS + .iter() + .cloned() + .map(|mut g| { + let mut tables = vec![]; + + let mut num_bits = 0; + while num_bits <= jubjub::Fr::NUM_BITS { + let mut table = Vec::with_capacity(1 << window); + let mut base = SubgroupPoint::identity(); + + for _ in 0..(1 << window) { + table.push(base); + base += g; + } + + tables.push(table); + num_bits += window; + + for _ in 0..window { + g = g.double(); + } + } + + tables + }) + .collect() +} + +#[cfg(test)] +mod tests { + use jubjub::SubgroupPoint; + + use super::*; + use crate::sapling::group_hash::group_hash; + + fn find_group_hash(m: &[u8], personalization: &[u8; 8]) -> SubgroupPoint { + let mut tag = m.to_vec(); + let i = tag.len(); + tag.push(0u8); + + loop { + let gh = group_hash(&tag, personalization); + + // We don't want to overflow and start reusing generators + assert!(tag[i] != u8::max_value()); + tag[i] += 1; + + if let Some(gh) = gh { + break gh; + } + } + } + + #[test] + fn proof_generation_key_base_generator() { + assert_eq!( + find_group_hash(&[], PROOF_GENERATION_KEY_BASE_GENERATOR_PERSONALIZATION), + PROOF_GENERATION_KEY_GENERATOR, + ); + } + + #[test] + fn note_commitment_randomness_generator() { + assert_eq!( + find_group_hash(b"r", PEDERSEN_HASH_GENERATORS_PERSONALIZATION), + NOTE_COMMITMENT_RANDOMNESS_GENERATOR, + ); + } + + #[test] + fn nullifier_position_generator() { + assert_eq!( + find_group_hash(&[], NULLIFIER_POSITION_IN_TREE_GENERATOR_PERSONALIZATION), + NULLIFIER_POSITION_GENERATOR, + ); + } + + #[test] + fn value_commitment_value_generator() { + assert_eq!( + find_group_hash(b"v", VALUE_COMMITMENT_GENERATOR_PERSONALIZATION), + VALUE_COMMITMENT_VALUE_GENERATOR, + ); + } + + #[test] + fn value_commitment_randomness_generator() { + assert_eq!( + find_group_hash(b"r", VALUE_COMMITMENT_GENERATOR_PERSONALIZATION), + VALUE_COMMITMENT_RANDOMNESS_GENERATOR, + ); + } + + #[test] + fn spending_key_generator() { + assert_eq!( + find_group_hash(&[], SPENDING_KEY_GENERATOR_PERSONALIZATION), + SPENDING_KEY_GENERATOR, + ); + } + + #[test] + fn pedersen_hash_generators() { + for (m, actual) in PEDERSEN_HASH_GENERATORS.iter().enumerate() { + assert_eq!( + &find_group_hash( + &(m as u32).to_le_bytes(), + PEDERSEN_HASH_GENERATORS_PERSONALIZATION + ), + actual + ); + } + } + + #[test] + fn no_duplicate_fixed_base_generators() { + let fixed_base_generators = [ + PROOF_GENERATION_KEY_GENERATOR, + NOTE_COMMITMENT_RANDOMNESS_GENERATOR, + NULLIFIER_POSITION_GENERATOR, + VALUE_COMMITMENT_VALUE_GENERATOR, + VALUE_COMMITMENT_RANDOMNESS_GENERATOR, + SPENDING_KEY_GENERATOR, + ]; + + // Check for duplicates, far worse than spec inconsistencies! + for (i, p1) in fixed_base_generators.iter().enumerate() { + if p1.is_identity().into() { + panic!("Neutral element!"); + } + + for p2 in fixed_base_generators.iter().skip(i + 1) { + if p1 == p2 { + panic!("Duplicate generator!"); + } + } + } + } + + /// Check for simple relations between the generators, that make finding collisions easy; + /// far worse than spec inconsistencies! + fn check_consistency_of_pedersen_hash_generators( + pedersen_hash_generators: &[jubjub::SubgroupPoint], + ) { + for (i, p1) in pedersen_hash_generators.iter().enumerate() { + if p1.is_identity().into() { + panic!("Neutral element!"); + } + for p2 in pedersen_hash_generators.iter().skip(i + 1) { + if p1 == p2 { + panic!("Duplicate generator!"); + } + if *p1 == -p2 { + panic!("Inverse generator!"); + } + } + + // check for a generator being the sum of any other two + for (j, p2) in pedersen_hash_generators.iter().enumerate() { + if j == i { + continue; + } + for (k, p3) in pedersen_hash_generators.iter().enumerate() { + if k == j || k == i { + continue; + } + let sum = p2 + p3; + if sum == *p1 { + panic!("Linear relation between generators!"); + } + } + } + } + } + + #[test] + fn pedersen_hash_generators_consistency() { + check_consistency_of_pedersen_hash_generators(PEDERSEN_HASH_GENERATORS); + } + + #[test] + #[should_panic(expected = "Linear relation between generators!")] + fn test_jubjub_bls12_pedersen_hash_generators_consistency_check_linear_relation() { + let mut pedersen_hash_generators = PEDERSEN_HASH_GENERATORS.to_vec(); + + // Test for linear relation + pedersen_hash_generators.push(PEDERSEN_HASH_GENERATORS[0] + PEDERSEN_HASH_GENERATORS[1]); + + check_consistency_of_pedersen_hash_generators(&pedersen_hash_generators); + } +} diff --git a/zcash_primitives/src/sapling/group_hash.rs b/zcash_primitives/src/sapling/group_hash.rs index 5a9f06a096..30b0c5d0db 100644 --- a/zcash_primitives/src/sapling/group_hash.rs +++ b/zcash_primitives/src/sapling/group_hash.rs @@ -5,7 +5,7 @@ use ff::PrimeField; use group::{cofactor::CofactorGroup, Group, GroupEncoding}; -use crate::constants; +use super::constants; use blake2s_simd::Params; /// Produces a random point in the Jubjub curve. diff --git a/zcash_primitives/src/sapling/keys.rs b/zcash_primitives/src/sapling/keys.rs index ad42694927..3fd458f490 100644 --- a/zcash_primitives/src/sapling/keys.rs +++ b/zcash_primitives/src/sapling/keys.rs @@ -8,6 +8,7 @@ use std::io::{self, Read, Write}; use super::{ address::PaymentAddress, + constants::{self, PROOF_GENERATION_KEY_GENERATOR, SPENDING_KEY_GENERATOR}, note_encryption::KDF_SAPLING_PERSONALIZATION, spec::{ crh_ivk, diversify_hash, ka_sapling_agree, ka_sapling_agree_prepared, @@ -15,10 +16,7 @@ use super::{ PreparedBaseSubgroup, PreparedScalar, }, }; -use crate::{ - constants::{self, PROOF_GENERATION_KEY_GENERATOR, SPENDING_KEY_GENERATOR}, - keys::prf_expand, -}; +use crate::keys::prf_expand; use blake2b_simd::{Hash as Blake2bHash, Params as Blake2bParams}; use ff::PrimeField; @@ -509,7 +507,7 @@ mod tests { use group::{Group, GroupEncoding}; use super::FullViewingKey; - use crate::constants::SPENDING_KEY_GENERATOR; + use crate::sapling::constants::SPENDING_KEY_GENERATOR; #[test] fn ak_must_be_prime_order() { diff --git a/zcash_primitives/src/sapling/pedersen_hash.rs b/zcash_primitives/src/sapling/pedersen_hash.rs index 0e5ed26c53..20cc40a5c8 100644 --- a/zcash_primitives/src/sapling/pedersen_hash.rs +++ b/zcash_primitives/src/sapling/pedersen_hash.rs @@ -8,7 +8,7 @@ use ff::PrimeField; use group::Group; use std::ops::{AddAssign, Neg}; -use crate::constants::{ +use super::constants::{ PEDERSEN_HASH_CHUNKS_PER_GENERATOR, PEDERSEN_HASH_EXP_TABLE, PEDERSEN_HASH_EXP_WINDOW_SIZE, }; diff --git a/zcash_primitives/src/sapling/prover.rs b/zcash_primitives/src/sapling/prover.rs index 0dc8c86816..3650d5458d 100644 --- a/zcash_primitives/src/sapling/prover.rs +++ b/zcash_primitives/src/sapling/prover.rs @@ -68,9 +68,9 @@ pub mod mock { use super::TxProver; use crate::{ - constants::SPENDING_KEY_GENERATOR, sapling::{ self, + constants::SPENDING_KEY_GENERATOR, redjubjub::{PublicKey, Signature}, value::{NoteValue, ValueCommitTrapdoor, ValueCommitment}, Diversifier, PaymentAddress, ProofGenerationKey, Rseed, diff --git a/zcash_primitives/src/sapling/redjubjub.rs b/zcash_primitives/src/sapling/redjubjub.rs index 695098681b..9df1d07290 100644 --- a/zcash_primitives/src/sapling/redjubjub.rs +++ b/zcash_primitives/src/sapling/redjubjub.rs @@ -228,7 +228,7 @@ mod tests { use rand_xorshift::XorShiftRng; use super::*; - use crate::constants::SPENDING_KEY_GENERATOR; + use crate::sapling::constants::SPENDING_KEY_GENERATOR; #[test] fn test_batch_verify() { diff --git a/zcash_primitives/src/sapling/spec.rs b/zcash_primitives/src/sapling/spec.rs index 0e78073265..eabf17a3d4 100644 --- a/zcash_primitives/src/sapling/spec.rs +++ b/zcash_primitives/src/sapling/spec.rs @@ -4,13 +4,13 @@ use blake2s_simd::Params as Blake2sParams; use group::{cofactor::CofactorGroup, ff::PrimeField, Curve, GroupEncoding, WnafBase, WnafScalar}; use super::{ + constants::{ + CRH_IVK_PERSONALIZATION, KEY_DIVERSIFICATION_PERSONALIZATION, + NOTE_COMMITMENT_RANDOMNESS_GENERATOR, NULLIFIER_POSITION_GENERATOR, PRF_NF_PERSONALIZATION, + }, group_hash::group_hash, pedersen_hash::{pedersen_hash, Personalization}, }; -use crate::constants::{ - CRH_IVK_PERSONALIZATION, KEY_DIVERSIFICATION_PERSONALIZATION, - NOTE_COMMITMENT_RANDOMNESS_GENERATOR, NULLIFIER_POSITION_GENERATOR, PRF_NF_PERSONALIZATION, -}; const PREPARED_WINDOW_SIZE: usize = 4; pub(crate) type PreparedBase = WnafBase; diff --git a/zcash_primitives/src/sapling/value.rs b/zcash_primitives/src/sapling/value.rs index b504fb0b72..ed68027dfa 100644 --- a/zcash_primitives/src/sapling/value.rs +++ b/zcash_primitives/src/sapling/value.rs @@ -43,7 +43,7 @@ use group::GroupEncoding; use rand::RngCore; use subtle::CtOption; -use crate::constants::{VALUE_COMMITMENT_RANDOMNESS_GENERATOR, VALUE_COMMITMENT_VALUE_GENERATOR}; +use super::constants::{VALUE_COMMITMENT_RANDOMNESS_GENERATOR, VALUE_COMMITMENT_VALUE_GENERATOR}; mod sums; pub use sums::{CommitmentSum, OverflowError, TrapdoorSum, ValueSum}; diff --git a/zcash_primitives/src/sapling/value/sums.rs b/zcash_primitives/src/sapling/value/sums.rs index 7a5420df25..f1d8e1a7f7 100644 --- a/zcash_primitives/src/sapling/value/sums.rs +++ b/zcash_primitives/src/sapling/value/sums.rs @@ -3,8 +3,7 @@ use core::iter::Sum; use core::ops::{Add, AddAssign, Sub, SubAssign}; use super::{NoteValue, ValueCommitTrapdoor, ValueCommitment}; -use crate::constants::VALUE_COMMITMENT_VALUE_GENERATOR; -use crate::sapling::redjubjub; +use crate::sapling::{constants::VALUE_COMMITMENT_VALUE_GENERATOR, redjubjub}; /// A value operation overflowed. #[derive(Debug)] diff --git a/zcash_primitives/src/transaction/components/sapling.rs b/zcash_primitives/src/transaction/components/sapling.rs index 149bfdbd4c..f899155e8c 100644 --- a/zcash_primitives/src/transaction/components/sapling.rs +++ b/zcash_primitives/src/transaction/components/sapling.rs @@ -715,8 +715,8 @@ pub mod testing { use rand::{rngs::StdRng, SeedableRng}; use crate::{ - constants::{SPENDING_KEY_GENERATOR, VALUE_COMMITMENT_RANDOMNESS_GENERATOR}, sapling::{ + constants::{SPENDING_KEY_GENERATOR, VALUE_COMMITMENT_RANDOMNESS_GENERATOR}, note::ExtractedNoteCommitment, redjubjub::{PrivateKey, PublicKey}, value::{ diff --git a/zcash_primitives/src/zip32/sapling.rs b/zcash_primitives/src/zip32/sapling.rs index 2ce8127591..cd5909ae3e 100644 --- a/zcash_primitives/src/zip32/sapling.rs +++ b/zcash_primitives/src/zip32/sapling.rs @@ -16,10 +16,12 @@ use super::{ Scope, ViewingKey, }; use crate::{ - constants::{PROOF_GENERATION_KEY_GENERATOR, SPENDING_KEY_GENERATOR}, keys::{prf_expand, prf_expand_vec}, - sapling::keys::{DecodingError, ExpandedSpendingKey, FullViewingKey, OutgoingViewingKey}, - sapling::SaplingIvk, + sapling::{ + constants::{PROOF_GENERATION_KEY_GENERATOR, SPENDING_KEY_GENERATOR}, + keys::{DecodingError, ExpandedSpendingKey, FullViewingKey, OutgoingViewingKey}, + SaplingIvk, + }, }; pub const ZIP32_SAPLING_MASTER_PERSONALIZATION: &[u8; 16] = b"ZcashIP32Sapling"; diff --git a/zcash_proofs/src/circuit/ecc.rs b/zcash_proofs/src/circuit/ecc.rs index ca797b3bff..3427bcd407 100644 --- a/zcash_proofs/src/circuit/ecc.rs +++ b/zcash_proofs/src/circuit/ecc.rs @@ -732,7 +732,7 @@ mod test { for _ in 0..100 { let mut cs = TestConstraintSystem::::new(); - let p = zcash_primitives::constants::NOTE_COMMITMENT_RANDOMNESS_GENERATOR; + let p = zcash_primitives::sapling::constants::NOTE_COMMITMENT_RANDOMNESS_GENERATOR; let s = jubjub::Fr::random(&mut rng); let q = jubjub::ExtendedPoint::from(p * s).to_affine(); let (u1, v1) = (q.get_u(), q.get_v()); diff --git a/zcash_proofs/src/circuit/sapling.rs b/zcash_proofs/src/circuit/sapling.rs index fddbaae073..a1cf2cf679 100644 --- a/zcash_proofs/src/circuit/sapling.rs +++ b/zcash_proofs/src/circuit/sapling.rs @@ -4,9 +4,7 @@ use group::{ff::PrimeField, Curve}; use bellman::{Circuit, ConstraintSystem, SynthesisError}; -use zcash_primitives::constants; - -use zcash_primitives::sapling::{PaymentAddress, ProofGenerationKey}; +use zcash_primitives::sapling::{constants, PaymentAddress, ProofGenerationKey}; use super::ecc; use super::pedersen_hash; diff --git a/zcash_proofs/src/constants.rs b/zcash_proofs/src/constants.rs index 658a5e43da..f4ea2d462c 100644 --- a/zcash_proofs/src/constants.rs +++ b/zcash_proofs/src/constants.rs @@ -4,7 +4,9 @@ use bls12_381::Scalar; use group::{ff::Field, Curve, Group}; use jubjub::ExtendedPoint; use lazy_static::lazy_static; -use zcash_primitives::constants::{PEDERSEN_HASH_CHUNKS_PER_GENERATOR, PEDERSEN_HASH_GENERATORS}; +use zcash_primitives::sapling::constants::{ + PEDERSEN_HASH_CHUNKS_PER_GENERATOR, PEDERSEN_HASH_GENERATORS, +}; /// The `d` constant of the twisted Edwards curve. pub(crate) const EDWARDS_D: Scalar = Scalar::from_raw([ @@ -42,22 +44,22 @@ pub type FixedGeneratorOwned = Vec>; lazy_static! { pub static ref PROOF_GENERATION_KEY_GENERATOR: FixedGeneratorOwned = - generate_circuit_generator(zcash_primitives::constants::PROOF_GENERATION_KEY_GENERATOR); + generate_circuit_generator(zcash_primitives::sapling::constants::PROOF_GENERATION_KEY_GENERATOR); pub static ref NOTE_COMMITMENT_RANDOMNESS_GENERATOR: FixedGeneratorOwned = - generate_circuit_generator(zcash_primitives::constants::NOTE_COMMITMENT_RANDOMNESS_GENERATOR); + generate_circuit_generator(zcash_primitives::sapling::constants::NOTE_COMMITMENT_RANDOMNESS_GENERATOR); pub static ref NULLIFIER_POSITION_GENERATOR: FixedGeneratorOwned = - generate_circuit_generator(zcash_primitives::constants::NULLIFIER_POSITION_GENERATOR); + generate_circuit_generator(zcash_primitives::sapling::constants::NULLIFIER_POSITION_GENERATOR); pub static ref VALUE_COMMITMENT_VALUE_GENERATOR: FixedGeneratorOwned = - generate_circuit_generator(zcash_primitives::constants::VALUE_COMMITMENT_VALUE_GENERATOR); + generate_circuit_generator(zcash_primitives::sapling::constants::VALUE_COMMITMENT_VALUE_GENERATOR); pub static ref VALUE_COMMITMENT_RANDOMNESS_GENERATOR: FixedGeneratorOwned = - generate_circuit_generator(zcash_primitives::constants::VALUE_COMMITMENT_RANDOMNESS_GENERATOR); + generate_circuit_generator(zcash_primitives::sapling::constants::VALUE_COMMITMENT_RANDOMNESS_GENERATOR); pub static ref SPENDING_KEY_GENERATOR: FixedGeneratorOwned = - generate_circuit_generator(zcash_primitives::constants::SPENDING_KEY_GENERATOR); + generate_circuit_generator(zcash_primitives::sapling::constants::SPENDING_KEY_GENERATOR); /// The pre-computed window tables `[-4, 3, 2, 1, 1, 2, 3, 4]` of different magnitudes /// of the Pedersen hash segment generators. diff --git a/zcash_proofs/src/sapling/prover.rs b/zcash_proofs/src/sapling/prover.rs index 9ecfb8f6a9..47716956fc 100644 --- a/zcash_proofs/src/sapling/prover.rs +++ b/zcash_proofs/src/sapling/prover.rs @@ -6,8 +6,8 @@ use bls12_381::Bls12; use group::{Curve, GroupEncoding}; use rand_core::OsRng; use zcash_primitives::{ - constants::{SPENDING_KEY_GENERATOR, VALUE_COMMITMENT_RANDOMNESS_GENERATOR}, sapling::{ + constants::{SPENDING_KEY_GENERATOR, VALUE_COMMITMENT_RANDOMNESS_GENERATOR}, redjubjub::{PublicKey, Signature}, value::{CommitmentSum, NoteValue, TrapdoorSum, ValueCommitTrapdoor, ValueCommitment}, Diversifier, MerklePath, Note, PaymentAddress, ProofGenerationKey, Rseed, diff --git a/zcash_proofs/src/sapling/verifier/single.rs b/zcash_proofs/src/sapling/verifier/single.rs index 02c1a9876f..94938374e9 100644 --- a/zcash_proofs/src/sapling/verifier/single.rs +++ b/zcash_proofs/src/sapling/verifier/single.rs @@ -1,8 +1,8 @@ use bellman::groth16::{verify_proof, PreparedVerifyingKey, Proof}; use bls12_381::Bls12; use zcash_primitives::{ - constants::{SPENDING_KEY_GENERATOR, VALUE_COMMITMENT_RANDOMNESS_GENERATOR}, sapling::{ + constants::{SPENDING_KEY_GENERATOR, VALUE_COMMITMENT_RANDOMNESS_GENERATOR}, note::ExtractedNoteCommitment, redjubjub::{PublicKey, Signature}, value::ValueCommitment, From db311050671b892d5385bfb2738e5981e2ab5f4c Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 29 Sep 2023 16:59:41 +0000 Subject: [PATCH 0214/1122] Move Sapling circuits from `zcash_proofs` to `zcash_primitives::sapling` Closes zcash/librustzcash#737. --- Cargo.lock | 4 +-- zcash_primitives/CHANGELOG.md | 2 ++ zcash_primitives/Cargo.toml | 6 ++++ .../benches/sapling_circuit.rs | 6 ++-- zcash_primitives/src/sapling.rs | 1 + .../src/sapling/circuit.rs | 36 ++++++++++--------- .../src/sapling/circuit}/constants.rs | 16 ++++----- .../src/sapling}/circuit/ecc.rs | 9 +++-- .../src/sapling}/circuit/pedersen_hash.rs | 6 ++-- zcash_proofs/CHANGELOG.md | 4 +++ zcash_proofs/Cargo.toml | 10 ------ zcash_proofs/src/circuit.rs | 4 --- zcash_proofs/src/lib.rs | 1 - zcash_proofs/src/sapling/prover.rs | 3 +- 14 files changed, 55 insertions(+), 53 deletions(-) rename zcash_proofs/benches/sapling.rs => zcash_primitives/benches/sapling_circuit.rs (95%) rename zcash_proofs/src/circuit/sapling.rs => zcash_primitives/src/sapling/circuit.rs (97%) rename {zcash_proofs/src => zcash_primitives/src/sapling/circuit}/constants.rs (88%) rename {zcash_proofs/src => zcash_primitives/src/sapling}/circuit/ecc.rs (99%) rename {zcash_proofs/src => zcash_primitives/src/sapling}/circuit/pedersen_hash.rs (98%) diff --git a/Cargo.lock b/Cargo.lock index 9fb8e2c0c2..01035a4151 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3079,6 +3079,7 @@ version = "0.13.0" dependencies = [ "aes", "assert_matches", + "bellman", "bip0039", "bitvec", "blake2b_simd", @@ -3121,16 +3122,13 @@ dependencies = [ "blake2b_simd", "bls12_381", "byteorder", - "criterion", "group", "home", "jubjub", "known-folders", "lazy_static", "minreq", - "pprof", "rand_core", - "rand_xorshift", "redjubjub", "tracing", "wagyu-zcash-parameters", diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index f7f326728b..da7ba6f92d 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -7,7 +7,9 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added +- Dependency on `bellman 0.14`. - `zcash_primitives::sapling`: + - `circuit` module (moved from `zcash_proofs::circuit::sapling`). - `constants` module. ### Removed diff --git a/zcash_primitives/Cargo.toml b/zcash_primitives/Cargo.toml index efcaa9b55d..f976ba3bbe 100644 --- a/zcash_primitives/Cargo.toml +++ b/zcash_primitives/Cargo.toml @@ -39,6 +39,7 @@ memuse.workspace = true subtle.workspace = true # - Shielded protocols +bellman = { version = "0.14", default-features = false, features = ["groth16"] } bls12_381.workspace = true ff.workspace = true group = { workspace = true, features = ["wnaf-memuse"] } @@ -116,5 +117,10 @@ harness = false name = "pedersen_hash" harness = false +[[bench]] +name = "sapling_circuit" +harness = false +required-features = ["local-prover"] + [badges] maintenance = { status = "actively-developed" } diff --git a/zcash_proofs/benches/sapling.rs b/zcash_primitives/benches/sapling_circuit.rs similarity index 95% rename from zcash_proofs/benches/sapling.rs rename to zcash_primitives/benches/sapling_circuit.rs index 1a1592bfd6..c35c3bd210 100644 --- a/zcash_proofs/benches/sapling.rs +++ b/zcash_primitives/benches/sapling_circuit.rs @@ -7,8 +7,10 @@ use criterion::Criterion; use group::{ff::Field, Group}; use rand_core::{RngCore, SeedableRng}; use rand_xorshift::XorShiftRng; -use zcash_primitives::sapling::{Diversifier, ProofGenerationKey}; -use zcash_proofs::circuit::sapling::{Spend, ValueCommitmentOpening}; +use zcash_primitives::sapling::{ + circuit::{Spend, ValueCommitmentOpening}, + Diversifier, ProofGenerationKey, +}; #[cfg(unix)] use pprof::criterion::{Output, PProfProfiler}; diff --git a/zcash_primitives/src/sapling.rs b/zcash_primitives/src/sapling.rs index 7cccd252f7..759826581c 100644 --- a/zcash_primitives/src/sapling.rs +++ b/zcash_primitives/src/sapling.rs @@ -1,6 +1,7 @@ //! Structs and constants specific to the Sapling shielded pool. mod address; +pub mod circuit; pub mod constants; pub mod group_hash; pub mod keys; diff --git a/zcash_proofs/src/circuit/sapling.rs b/zcash_primitives/src/sapling/circuit.rs similarity index 97% rename from zcash_proofs/src/circuit/sapling.rs rename to zcash_primitives/src/sapling/circuit.rs index a1cf2cf679..78ef9b8472 100644 --- a/zcash_proofs/src/circuit/sapling.rs +++ b/zcash_primitives/src/sapling/circuit.rs @@ -4,26 +4,29 @@ use group::{ff::PrimeField, Curve}; use bellman::{Circuit, ConstraintSystem, SynthesisError}; -use zcash_primitives::sapling::{constants, PaymentAddress, ProofGenerationKey}; +use super::{PaymentAddress, ProofGenerationKey}; -use super::ecc; -use super::pedersen_hash; -use crate::constants::{ - NOTE_COMMITMENT_RANDOMNESS_GENERATOR, NULLIFIER_POSITION_GENERATOR, - PROOF_GENERATION_KEY_GENERATOR, SPENDING_KEY_GENERATOR, VALUE_COMMITMENT_RANDOMNESS_GENERATOR, - VALUE_COMMITMENT_VALUE_GENERATOR, -}; use bellman::gadgets::blake2s; use bellman::gadgets::boolean; use bellman::gadgets::multipack; use bellman::gadgets::num; use bellman::gadgets::Assignment; +use self::constants::{ + NOTE_COMMITMENT_RANDOMNESS_GENERATOR, NULLIFIER_POSITION_GENERATOR, + PROOF_GENERATION_KEY_GENERATOR, SPENDING_KEY_GENERATOR, VALUE_COMMITMENT_RANDOMNESS_GENERATOR, + VALUE_COMMITMENT_VALUE_GENERATOR, +}; + #[cfg(test)] use group::ff::PrimeFieldBits; #[cfg(test)] -use zcash_primitives::sapling::value::NoteValue; +use super::value::NoteValue; + +mod constants; +mod ecc; +mod pedersen_hash; /// The opening (value and randomness) of a Sapling value commitment. #[derive(Clone)] @@ -35,8 +38,9 @@ pub struct ValueCommitmentOpening { #[cfg(test)] impl ValueCommitmentOpening { fn commitment(&self) -> jubjub::ExtendedPoint { - let cv = (constants::VALUE_COMMITMENT_VALUE_GENERATOR * jubjub::Fr::from(self.value)) - + (constants::VALUE_COMMITMENT_RANDOMNESS_GENERATOR * self.randomness); + let cv = (super::constants::VALUE_COMMITMENT_VALUE_GENERATOR + * jubjub::Fr::from(self.value)) + + (super::constants::VALUE_COMMITMENT_RANDOMNESS_GENERATOR * self.randomness); cv.into() } } @@ -207,7 +211,7 @@ impl Circuit for Spend { let mut ivk = blake2s::blake2s( cs.namespace(|| "computation of ivk"), &ivk_preimage, - constants::CRH_IVK_PERSONALIZATION, + super::constants::CRH_IVK_PERSONALIZATION, )?; // drop_5 to ensure it's in the field @@ -400,7 +404,7 @@ impl Circuit for Spend { let nf = blake2s::blake2s( cs.namespace(|| "nf computation"), &nf_preimage, - constants::PRF_NF_PERSONALIZATION, + super::constants::PRF_NF_PERSONALIZATION, )?; multipack::pack_into_inputs(cs.namespace(|| "pack nullifier"), &nf) @@ -534,11 +538,11 @@ impl Circuit for Output { #[test] fn test_input_circuit_with_bls12_381() { + use crate::sapling::{pedersen_hash, Diversifier, Note, ProofGenerationKey, Rseed}; use bellman::gadgets::test::*; use group::{ff::Field, Group}; use rand_core::{RngCore, SeedableRng}; use rand_xorshift::XorShiftRng; - use zcash_primitives::sapling::{pedersen_hash, Diversifier, Note, ProofGenerationKey, Rseed}; let mut rng = XorShiftRng::from_seed([ 0x58, 0x62, 0xbe, 0x3d, 0x76, 0x3d, 0x31, 0x8d, 0x17, 0xdb, 0x37, 0x32, 0x54, 0x06, 0xbc, @@ -680,11 +684,11 @@ fn test_input_circuit_with_bls12_381() { #[test] fn test_input_circuit_with_bls12_381_external_test_vectors() { + use crate::sapling::{pedersen_hash, Diversifier, Note, ProofGenerationKey, Rseed}; use bellman::gadgets::test::*; use group::{ff::Field, Group}; use rand_core::{RngCore, SeedableRng}; use rand_xorshift::XorShiftRng; - use zcash_primitives::sapling::{pedersen_hash, Diversifier, Note, ProofGenerationKey, Rseed}; let mut rng = XorShiftRng::from_seed([ 0x59, 0x62, 0xbe, 0x3d, 0x76, 0x3d, 0x31, 0x8d, 0x17, 0xdb, 0x37, 0x32, 0x54, 0x06, 0xbc, @@ -860,11 +864,11 @@ fn test_input_circuit_with_bls12_381_external_test_vectors() { #[test] fn test_output_circuit_with_bls12_381() { + use crate::sapling::{Diversifier, ProofGenerationKey, Rseed}; use bellman::gadgets::test::*; use group::{ff::Field, Group}; use rand_core::{RngCore, SeedableRng}; use rand_xorshift::XorShiftRng; - use zcash_primitives::sapling::{Diversifier, ProofGenerationKey, Rseed}; let mut rng = XorShiftRng::from_seed([ 0x58, 0x62, 0xbe, 0x3d, 0x76, 0x3d, 0x31, 0x8d, 0x17, 0xdb, 0x37, 0x32, 0x54, 0x06, 0xbc, diff --git a/zcash_proofs/src/constants.rs b/zcash_primitives/src/sapling/circuit/constants.rs similarity index 88% rename from zcash_proofs/src/constants.rs rename to zcash_primitives/src/sapling/circuit/constants.rs index f4ea2d462c..d7afaeaa32 100644 --- a/zcash_proofs/src/constants.rs +++ b/zcash_primitives/src/sapling/circuit/constants.rs @@ -1,12 +1,10 @@ //! Various constants used for the Zcash proofs. +use crate::sapling::constants::{PEDERSEN_HASH_CHUNKS_PER_GENERATOR, PEDERSEN_HASH_GENERATORS}; use bls12_381::Scalar; use group::{ff::Field, Curve, Group}; use jubjub::ExtendedPoint; use lazy_static::lazy_static; -use zcash_primitives::sapling::constants::{ - PEDERSEN_HASH_CHUNKS_PER_GENERATOR, PEDERSEN_HASH_GENERATORS, -}; /// The `d` constant of the twisted Edwards curve. pub(crate) const EDWARDS_D: Scalar = Scalar::from_raw([ @@ -44,22 +42,22 @@ pub type FixedGeneratorOwned = Vec>; lazy_static! { pub static ref PROOF_GENERATION_KEY_GENERATOR: FixedGeneratorOwned = - generate_circuit_generator(zcash_primitives::sapling::constants::PROOF_GENERATION_KEY_GENERATOR); + generate_circuit_generator(crate::sapling::constants::PROOF_GENERATION_KEY_GENERATOR); pub static ref NOTE_COMMITMENT_RANDOMNESS_GENERATOR: FixedGeneratorOwned = - generate_circuit_generator(zcash_primitives::sapling::constants::NOTE_COMMITMENT_RANDOMNESS_GENERATOR); + generate_circuit_generator(crate::sapling::constants::NOTE_COMMITMENT_RANDOMNESS_GENERATOR); pub static ref NULLIFIER_POSITION_GENERATOR: FixedGeneratorOwned = - generate_circuit_generator(zcash_primitives::sapling::constants::NULLIFIER_POSITION_GENERATOR); + generate_circuit_generator(crate::sapling::constants::NULLIFIER_POSITION_GENERATOR); pub static ref VALUE_COMMITMENT_VALUE_GENERATOR: FixedGeneratorOwned = - generate_circuit_generator(zcash_primitives::sapling::constants::VALUE_COMMITMENT_VALUE_GENERATOR); + generate_circuit_generator(crate::sapling::constants::VALUE_COMMITMENT_VALUE_GENERATOR); pub static ref VALUE_COMMITMENT_RANDOMNESS_GENERATOR: FixedGeneratorOwned = - generate_circuit_generator(zcash_primitives::sapling::constants::VALUE_COMMITMENT_RANDOMNESS_GENERATOR); + generate_circuit_generator(crate::sapling::constants::VALUE_COMMITMENT_RANDOMNESS_GENERATOR); pub static ref SPENDING_KEY_GENERATOR: FixedGeneratorOwned = - generate_circuit_generator(zcash_primitives::sapling::constants::SPENDING_KEY_GENERATOR); + generate_circuit_generator(crate::sapling::constants::SPENDING_KEY_GENERATOR); /// The pre-computed window tables `[-4, 3, 2, 1, 1, 2, 3, 4]` of different magnitudes /// of the Pedersen hash segment generators. diff --git a/zcash_proofs/src/circuit/ecc.rs b/zcash_primitives/src/sapling/circuit/ecc.rs similarity index 99% rename from zcash_proofs/src/circuit/ecc.rs rename to zcash_primitives/src/sapling/circuit/ecc.rs index 3427bcd407..b61f5e222e 100644 --- a/zcash_proofs/src/circuit/ecc.rs +++ b/zcash_primitives/src/sapling/circuit/ecc.rs @@ -14,7 +14,7 @@ use bellman::gadgets::boolean::Boolean; use group::Curve; -use crate::constants::{FixedGenerator, EDWARDS_D, MONTGOMERY_A, MONTGOMERY_SCALE}; +use super::constants::{FixedGenerator, EDWARDS_D, MONTGOMERY_A, MONTGOMERY_SCALE}; #[derive(Clone)] pub struct EdwardsPoint { @@ -77,6 +77,7 @@ impl EdwardsPoint { &self.u } + #[cfg(test)] pub fn get_v(&self) -> &AllocatedNum { &self.v } @@ -630,7 +631,9 @@ mod test { use bellman::gadgets::test::*; use super::{fixed_base_multiplication, AllocatedNum, EdwardsPoint, MontgomeryPoint}; - use crate::constants::{to_montgomery_coords, NOTE_COMMITMENT_RANDOMNESS_GENERATOR}; + use crate::sapling::circuit::constants::{ + to_montgomery_coords, NOTE_COMMITMENT_RANDOMNESS_GENERATOR, + }; use bellman::gadgets::boolean::{AllocatedBit, Boolean}; #[test] @@ -732,7 +735,7 @@ mod test { for _ in 0..100 { let mut cs = TestConstraintSystem::::new(); - let p = zcash_primitives::sapling::constants::NOTE_COMMITMENT_RANDOMNESS_GENERATOR; + let p = crate::sapling::constants::NOTE_COMMITMENT_RANDOMNESS_GENERATOR; let s = jubjub::Fr::random(&mut rng); let q = jubjub::ExtendedPoint::from(p * s).to_affine(); let (u1, v1) = (q.get_u(), q.get_v()); diff --git a/zcash_proofs/src/circuit/pedersen_hash.rs b/zcash_primitives/src/sapling/circuit/pedersen_hash.rs similarity index 98% rename from zcash_proofs/src/circuit/pedersen_hash.rs rename to zcash_primitives/src/sapling/circuit/pedersen_hash.rs index 44f2923aa4..eee6666649 100644 --- a/zcash_proofs/src/circuit/pedersen_hash.rs +++ b/zcash_primitives/src/sapling/circuit/pedersen_hash.rs @@ -1,12 +1,12 @@ //! Gadget for Zcash's Pedersen hash. use super::ecc::{EdwardsPoint, MontgomeryPoint}; +pub use crate::sapling::pedersen_hash::Personalization; use bellman::gadgets::boolean::Boolean; use bellman::gadgets::lookup::*; use bellman::{ConstraintSystem, SynthesisError}; -pub use zcash_primitives::sapling::pedersen_hash::Personalization; -use crate::constants::PEDERSEN_CIRCUIT_GENERATORS; +use super::constants::PEDERSEN_CIRCUIT_GENERATORS; fn get_constant_bools(person: &Personalization) -> Vec { person @@ -105,12 +105,12 @@ where #[cfg(test)] mod test { use super::*; + use crate::sapling::pedersen_hash; use bellman::gadgets::boolean::{AllocatedBit, Boolean}; use bellman::gadgets::test::*; use group::{ff::PrimeField, Curve}; use rand_core::{RngCore, SeedableRng}; use rand_xorshift::XorShiftRng; - use zcash_primitives::sapling::pedersen_hash; /// Predict the number of constraints of a Pedersen hash fn ph_num_constraints(input_bits: usize) -> usize { diff --git a/zcash_proofs/CHANGELOG.md b/zcash_proofs/CHANGELOG.md index 41872dd149..33c36b7be6 100644 --- a/zcash_proofs/CHANGELOG.md +++ b/zcash_proofs/CHANGELOG.md @@ -6,6 +6,10 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Removed +- `zcash_proofs::circuit::sapling` (moved to `zcash_primitives::sapling::circuit`). +- `zcash_proofs::circuit::{ecc, pedersen_hash}` +- `zcash_proofs::constants` ## [0.13.0] - 2023-09-25 ### Changed diff --git a/zcash_proofs/Cargo.toml b/zcash_proofs/Cargo.toml index 694c5f4482..6136d54935 100644 --- a/zcash_proofs/Cargo.toml +++ b/zcash_proofs/Cargo.toml @@ -41,11 +41,6 @@ xdg = { version = "2.5", optional = true } [dev-dependencies] byteorder.workspace = true -criterion.workspace = true -rand_xorshift.workspace = true - -[target.'cfg(unix)'.dev-dependencies] -pprof = { version = "0.11", features = ["criterion", "flamegraph"] } # MSRV 1.56 [features] default = ["local-prover", "multicore"] @@ -58,11 +53,6 @@ multicore = ["bellman/multicore", "zcash_primitives/multicore"] [lib] bench = false -[[bench]] -name = "sapling" -harness = false -required-features = ["local-prover"] - [[example]] name = "get-params-path" required-features = ["directories"] diff --git a/zcash_proofs/src/circuit.rs b/zcash_proofs/src/circuit.rs index 6d26fa485e..0897b120d4 100644 --- a/zcash_proofs/src/circuit.rs +++ b/zcash_proofs/src/circuit.rs @@ -1,7 +1,3 @@ //! Implementations of the Zcash circuits and Zcash-specific gadgets. -pub mod ecc; -pub mod pedersen_hash; - -pub mod sapling; pub mod sprout; diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index a666f6e513..992b3a7375 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -19,7 +19,6 @@ use std::path::Path; use std::path::PathBuf; pub mod circuit; -pub mod constants; mod hashreader; pub mod sapling; pub mod sprout; diff --git a/zcash_proofs/src/sapling/prover.rs b/zcash_proofs/src/sapling/prover.rs index 47716956fc..05767f348b 100644 --- a/zcash_proofs/src/sapling/prover.rs +++ b/zcash_proofs/src/sapling/prover.rs @@ -7,6 +7,7 @@ use group::{Curve, GroupEncoding}; use rand_core::OsRng; use zcash_primitives::{ sapling::{ + circuit::{Output, Spend, ValueCommitmentOpening}, constants::{SPENDING_KEY_GENERATOR, VALUE_COMMITMENT_RANDOMNESS_GENERATOR}, redjubjub::{PublicKey, Signature}, value::{CommitmentSum, NoteValue, TrapdoorSum, ValueCommitTrapdoor, ValueCommitment}, @@ -15,8 +16,6 @@ use zcash_primitives::{ transaction::components::Amount, }; -use crate::circuit::sapling::{Output, Spend, ValueCommitmentOpening}; - /// A context object for creating the Sapling components of a Zcash transaction. pub struct SaplingProvingContext { bsk: TrapdoorSum, From b5e24751d65b322e3e2911cc1334f503e8174c52 Mon Sep 17 00:00:00 2001 From: zancas Date: Fri, 29 Sep 2023 12:55:06 -0600 Subject: [PATCH 0215/1122] add Debug to transaction builder-reported types --- zcash_client_backend/src/fees.rs | 1 + .../src/transaction/components/transparent/builder.rs | 1 + .../src/transaction/components/transparent/fees.rs | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index b83477159e..050f3dbfa1 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -219,6 +219,7 @@ pub(crate) mod tests { transparent::{fees as transparent, OutPoint, TxOut}, }; + #[derive(Debug)] pub(crate) struct TestTransparentInput { pub outpoint: OutPoint, pub coin: TxOut, diff --git a/zcash_primitives/src/transaction/components/transparent/builder.rs b/zcash_primitives/src/transaction/components/transparent/builder.rs index a65645fc93..95a8b89491 100644 --- a/zcash_primitives/src/transaction/components/transparent/builder.rs +++ b/zcash_primitives/src/transaction/components/transparent/builder.rs @@ -43,6 +43,7 @@ impl fmt::Display for Error { /// An uninhabited type that allows the type of [`TransparentBuilder::inputs`] /// to resolve when the transparent-inputs feature is not turned on. #[cfg(not(feature = "transparent-inputs"))] +#[derive(Debug)] enum InvalidTransparentInput {} #[cfg(not(feature = "transparent-inputs"))] diff --git a/zcash_primitives/src/transaction/components/transparent/fees.rs b/zcash_primitives/src/transaction/components/transparent/fees.rs index 1e63e16c33..4b3f4ddb01 100644 --- a/zcash_primitives/src/transaction/components/transparent/fees.rs +++ b/zcash_primitives/src/transaction/components/transparent/fees.rs @@ -9,7 +9,7 @@ use crate::{ /// This trait provides a minimized view of a transparent input suitable for use in /// fee and change computation. -pub trait InputView { +pub trait InputView: std::fmt::Debug { /// The outpoint to which the input refers. fn outpoint(&self) -> &OutPoint; /// The previous output being spent. @@ -18,7 +18,7 @@ pub trait InputView { /// This trait provides a minimized view of a transparent output suitable for use in /// fee and change computation. -pub trait OutputView { +pub trait OutputView: std::fmt::Debug { /// Returns the value of the output being created. fn value(&self) -> Amount; /// Returns the script corresponding to the newly created output. From 2bfeef9430c6ab5c0b188c574325dff09e61f066 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 2 Oct 2023 13:33:33 +0000 Subject: [PATCH 0216/1122] zcash_proofs: Remove immediate verification of created Spend proofs It can be useful to verify proofs after they have been created, but we were only doing this for spend proofs, not output proofs. It also duplicated code from the verifier logic. Once the prover and verifier have been refactored, it will be easier to just call the verifier immediately after the prover. --- zcash_proofs/CHANGELOG.md | 6 ++++ zcash_proofs/src/prover.rs | 4 ++- zcash_proofs/src/sapling/prover.rs | 45 ++---------------------------- 3 files changed, 11 insertions(+), 44 deletions(-) diff --git a/zcash_proofs/CHANGELOG.md b/zcash_proofs/CHANGELOG.md index 33c36b7be6..a35ba56690 100644 --- a/zcash_proofs/CHANGELOG.md +++ b/zcash_proofs/CHANGELOG.md @@ -6,6 +6,12 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- `zcash_proofs::sapling::prover`: + - The `verifying_key` argument `SaplingProvingContext::spend_proof` has been + removed. Callers should instead use `SaplingVerifyingContext` to verify + proofs after they have been created. + ### Removed - `zcash_proofs::circuit::sapling` (moved to `zcash_primitives::sapling::circuit`). - `zcash_proofs::circuit::{ecc, pedersen_hash}` diff --git a/zcash_proofs/src/prover.rs b/zcash_proofs/src/prover.rs index 63c1db79f2..40b1240fba 100644 --- a/zcash_proofs/src/prover.rs +++ b/zcash_proofs/src/prover.rs @@ -22,6 +22,9 @@ use crate::{default_params_folder, SAPLING_OUTPUT_NAME, SAPLING_SPEND_NAME}; /// locally-accessible paths. pub struct LocalTxProver { spend_params: Parameters, + // TODO: Either re-introduce verification-after-proving (once the verifier is + // refactored), or remove this. + #[allow(unused)] spend_vk: PreparedVerifyingKey, output_params: Parameters, } @@ -164,7 +167,6 @@ impl TxProver for LocalTxProver { anchor, merkle_path, &self.spend_params, - &self.spend_vk, )?; let mut zkproof = [0u8; GROTH_PROOF_SIZE]; diff --git a/zcash_proofs/src/sapling/prover.rs b/zcash_proofs/src/sapling/prover.rs index 05767f348b..9c31efc9f7 100644 --- a/zcash_proofs/src/sapling/prover.rs +++ b/zcash_proofs/src/sapling/prover.rs @@ -1,9 +1,6 @@ -use bellman::{ - gadgets::multipack, - groth16::{create_random_proof, verify_proof, Parameters, PreparedVerifyingKey, Proof}, -}; +use bellman::groth16::{create_random_proof, Parameters, Proof}; use bls12_381::Bls12; -use group::{Curve, GroupEncoding}; +use group::GroupEncoding; use rand_core::OsRng; use zcash_primitives::{ sapling::{ @@ -52,7 +49,6 @@ impl SaplingProvingContext { anchor: bls12_381::Scalar, merkle_path: MerklePath, proving_key: &Parameters, - verifying_key: &PreparedVerifyingKey, ) -> Result<(Proof, ValueCommitment, PublicKey), ()> { // Initialize secure RNG let mut rng = OsRng; @@ -82,12 +78,6 @@ impl SaplingProvingContext { // Let's compute the nullifier while we have the position let note = Note::from_parts(payment_address, NoteValue::from_raw(value), rseed); - let nullifier = note.nf( - &viewing_key.nk, - u64::try_from(merkle_path.position()) - .expect("Sapling note commitment tree position must fit into a u64"), - ); - // We now have the full witness for our circuit let pos: u64 = merkle_path.position().into(); let instance = Spend { @@ -109,37 +99,6 @@ impl SaplingProvingContext { let proof = create_random_proof(instance, proving_key, &mut rng).expect("proving should not fail"); - // Try to verify the proof: - // Construct public input for circuit - let mut public_input = [bls12_381::Scalar::zero(); 7]; - { - let affine = rk.0.to_affine(); - let (u, v) = (affine.get_u(), affine.get_v()); - public_input[0] = u; - public_input[1] = v; - } - { - let affine = value_commitment.as_inner().to_affine(); - let (u, v) = (affine.get_u(), affine.get_v()); - public_input[2] = u; - public_input[3] = v; - } - public_input[4] = anchor; - - // Add the nullifier through multiscalar packing - { - let nullifier = multipack::bytes_to_bits_le(&nullifier.0); - let nullifier = multipack::compute_multipacking(&nullifier); - - assert_eq!(nullifier.len(), 2); - - public_input[5] = nullifier[0]; - public_input[6] = nullifier[1]; - } - - // Verify the proof - verify_proof(verifying_key, &proof, &public_input[..]).map_err(|_| ())?; - // Accumulate the value commitment in the context self.cv_sum += &value_commitment; From ea0fed39eb91a3727c47e12cc8cab1bc812a2cda Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 2 Oct 2023 14:03:09 +0000 Subject: [PATCH 0217/1122] zcash_proofs: Introduce newtype wrappers for Sapling parameters --- zcash_proofs/CHANGELOG.md | 9 +++++++++ zcash_proofs/src/lib.rs | 14 ++++++++++---- zcash_proofs/src/prover.rs | 11 +++++++---- zcash_proofs/src/sapling/prover.rs | 16 +++++++++------- 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/zcash_proofs/CHANGELOG.md b/zcash_proofs/CHANGELOG.md index a35ba56690..f544343b62 100644 --- a/zcash_proofs/CHANGELOG.md +++ b/zcash_proofs/CHANGELOG.md @@ -6,7 +6,16 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- `zcash_proofs::{SpendParameters, OutputParameters}` + ### Changed +- The new `SpendParameters` and `OutputParameters` types are used in the + following places: + - `zcash_proofs::ZcashParameters::{spend_params, output_params}` fields. + - `zcash_proofs::sapling::prover`: + - `SaplingProvingContext::{spend_proof, output_proof}` (the `proving_key` + arguments). - `zcash_proofs::sapling::prover`: - The `verifying_key` argument `SaplingProvingContext::spend_proof` has been removed. Callers should instead use `SaplingVerifyingContext` to verify diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 992b3a7375..bdd452bd44 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -283,11 +283,17 @@ fn stream_params_downloads_to_disk( Ok(()) } +/// The parameters for the Sapling Spend circuit. +pub struct SpendParameters(Parameters); + +/// The parameters for the Sapling Output circuit. +pub struct OutputParameters(Parameters); + /// Zcash Sprout and Sapling groth16 circuit parameters. pub struct ZcashParameters { - pub spend_params: Parameters, + pub spend_params: SpendParameters, pub spend_vk: PreparedVerifyingKey, - pub output_params: Parameters, + pub output_params: OutputParameters, pub output_vk: PreparedVerifyingKey, pub sprout_vk: Option>, } @@ -429,9 +435,9 @@ pub fn parse_parameters( let sprout_vk = sprout_vk.map(|vk| prepare_verifying_key(&vk)); ZcashParameters { - spend_params, + spend_params: SpendParameters(spend_params), spend_vk, - output_params, + output_params: OutputParameters(output_params), output_vk, sprout_vk, } diff --git a/zcash_proofs/src/prover.rs b/zcash_proofs/src/prover.rs index 40b1240fba..8bbd1abff6 100644 --- a/zcash_proofs/src/prover.rs +++ b/zcash_proofs/src/prover.rs @@ -1,6 +1,6 @@ //! Abstractions over the proving system and parameters for ease of use. -use bellman::groth16::{Parameters, PreparedVerifyingKey}; +use bellman::groth16::PreparedVerifyingKey; use bls12_381::Bls12; use std::path::Path; use zcash_primitives::{ @@ -13,7 +13,10 @@ use zcash_primitives::{ transaction::components::{Amount, GROTH_PROOF_SIZE}, }; -use crate::{load_parameters, parse_parameters, sapling::SaplingProvingContext}; +use crate::{ + load_parameters, parse_parameters, sapling::SaplingProvingContext, OutputParameters, + SpendParameters, +}; #[cfg(feature = "local-prover")] use crate::{default_params_folder, SAPLING_OUTPUT_NAME, SAPLING_SPEND_NAME}; @@ -21,12 +24,12 @@ use crate::{default_params_folder, SAPLING_OUTPUT_NAME, SAPLING_SPEND_NAME}; /// An implementation of [`TxProver`] using Sapling Spend and Output parameters from /// locally-accessible paths. pub struct LocalTxProver { - spend_params: Parameters, + spend_params: SpendParameters, // TODO: Either re-introduce verification-after-proving (once the verifier is // refactored), or remove this. #[allow(unused)] spend_vk: PreparedVerifyingKey, - output_params: Parameters, + output_params: OutputParameters, } impl LocalTxProver { diff --git a/zcash_proofs/src/sapling/prover.rs b/zcash_proofs/src/sapling/prover.rs index 9c31efc9f7..4352901084 100644 --- a/zcash_proofs/src/sapling/prover.rs +++ b/zcash_proofs/src/sapling/prover.rs @@ -1,4 +1,4 @@ -use bellman::groth16::{create_random_proof, Parameters, Proof}; +use bellman::groth16::{create_random_proof, Proof}; use bls12_381::Bls12; use group::GroupEncoding; use rand_core::OsRng; @@ -13,6 +13,8 @@ use zcash_primitives::{ transaction::components::Amount, }; +use crate::{OutputParameters, SpendParameters}; + /// A context object for creating the Sapling components of a Zcash transaction. pub struct SaplingProvingContext { bsk: TrapdoorSum, @@ -48,7 +50,7 @@ impl SaplingProvingContext { value: u64, anchor: bls12_381::Scalar, merkle_path: MerklePath, - proving_key: &Parameters, + proving_key: &SpendParameters, ) -> Result<(Proof, ValueCommitment, PublicKey), ()> { // Initialize secure RNG let mut rng = OsRng; @@ -96,8 +98,8 @@ impl SaplingProvingContext { }; // Create proof - let proof = - create_random_proof(instance, proving_key, &mut rng).expect("proving should not fail"); + let proof = create_random_proof(instance, &proving_key.0, &mut rng) + .expect("proving should not fail"); // Accumulate the value commitment in the context self.cv_sum += &value_commitment; @@ -114,7 +116,7 @@ impl SaplingProvingContext { payment_address: PaymentAddress, rcm: jubjub::Fr, value: u64, - proving_key: &Parameters, + proving_key: &OutputParameters, ) -> (Proof, ValueCommitment) { // Initialize secure RNG let mut rng = OsRng; @@ -143,8 +145,8 @@ impl SaplingProvingContext { }; // Create proof - let proof = - create_random_proof(instance, proving_key, &mut rng).expect("proving should not fail"); + let proof = create_random_proof(instance, &proving_key.0, &mut rng) + .expect("proving should not fail"); // Accumulate the value commitment in the context. We do this to check internal consistency. self.cv_sum -= &value_commitment; // Outputs subtract from the total. From 290bfa8b3143dba6054fa79d3f5b9bbffcecdec6 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 2 Oct 2023 14:16:43 +0000 Subject: [PATCH 0218/1122] zcash_primitives: Introduce a `SpendProver` trait --- zcash_primitives/CHANGELOG.md | 1 + zcash_primitives/src/sapling/prover.rs | 35 ++++++++- zcash_proofs/src/sapling/prover.rs | 99 +++++++++++++++++--------- 3 files changed, 101 insertions(+), 34 deletions(-) diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index da7ba6f92d..756d67ef37 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -11,6 +11,7 @@ and this library adheres to Rust's notion of - `zcash_primitives::sapling`: - `circuit` module (moved from `zcash_proofs::circuit::sapling`). - `constants` module. + - `prover::SpendProver` ### Removed - `zcash_primitives::constants`: diff --git a/zcash_primitives/src/sapling/prover.rs b/zcash_primitives/src/sapling/prover.rs index 3650d5458d..dcf0d65058 100644 --- a/zcash_primitives/src/sapling/prover.rs +++ b/zcash_primitives/src/sapling/prover.rs @@ -1,16 +1,49 @@ //! Abstractions over the proving system and parameters. +use rand_core::RngCore; + use crate::{ sapling::{ self, redjubjub::{PublicKey, Signature}, - value::ValueCommitment, + value::{NoteValue, ValueCommitTrapdoor, ValueCommitment}, + MerklePath, }, transaction::components::{Amount, GROTH_PROOF_SIZE}, }; use super::{Diversifier, PaymentAddress, ProofGenerationKey, Rseed}; +/// Interface for creating Sapling Spend proofs. +pub trait SpendProver { + /// The proof type created by this prover. + type Proof; + + /// Prepares an instance of the Sapling Spend circuit for the given inputs. + /// + /// Returns `None` if `diversifier` is not a valid Sapling diversifier. + #[allow(clippy::too_many_arguments)] + fn prepare_circuit( + proof_generation_key: ProofGenerationKey, + diversifier: Diversifier, + rseed: Rseed, + value: NoteValue, + alpha: jubjub::Fr, + rcv: ValueCommitTrapdoor, + anchor: bls12_381::Scalar, + merkle_path: MerklePath, + ) -> Option; + + /// Create the proof for a Sapling [`SpendDescription`]. + /// + /// [`SpendDescription`]: crate::transaction::components::SpendDescription + fn create_proof( + &self, + circuit: sapling::circuit::Spend, + rng: &mut R, + ) -> Self::Proof; +} + /// Interface for creating zero-knowledge proofs for shielded transactions. pub trait TxProver { /// Type for persisting any necessary context across multiple Sapling proofs. diff --git a/zcash_proofs/src/sapling/prover.rs b/zcash_proofs/src/sapling/prover.rs index 4352901084..6398f61190 100644 --- a/zcash_proofs/src/sapling/prover.rs +++ b/zcash_proofs/src/sapling/prover.rs @@ -1,11 +1,12 @@ use bellman::groth16::{create_random_proof, Proof}; use bls12_381::Bls12; use group::GroupEncoding; -use rand_core::OsRng; +use rand_core::{OsRng, RngCore}; use zcash_primitives::{ sapling::{ circuit::{Output, Spend, ValueCommitmentOpening}, constants::{SPENDING_KEY_GENERATOR, VALUE_COMMITMENT_RANDOMNESS_GENERATOR}, + prover::SpendProver, redjubjub::{PublicKey, Signature}, value::{CommitmentSum, NoteValue, TrapdoorSum, ValueCommitTrapdoor, ValueCommitment}, Diversifier, MerklePath, Note, PaymentAddress, ProofGenerationKey, Rseed, @@ -15,6 +16,56 @@ use zcash_primitives::{ use crate::{OutputParameters, SpendParameters}; +impl SpendProver for SpendParameters { + type Proof = Proof; + + fn prepare_circuit( + proof_generation_key: ProofGenerationKey, + diversifier: Diversifier, + rseed: Rseed, + value: NoteValue, + alpha: jubjub::Fr, + rcv: ValueCommitTrapdoor, + anchor: bls12_381::Scalar, + merkle_path: MerklePath, + ) -> Option { + // Construct the value commitment + let value_commitment_opening = ValueCommitmentOpening { + value: value.inner(), + randomness: rcv.inner(), + }; + + // Construct the viewing key + let viewing_key = proof_generation_key.to_viewing_key(); + + // Construct the payment address with the viewing key / diversifier + let payment_address = viewing_key.to_payment_address(diversifier)?; + + let note = Note::from_parts(payment_address, value, rseed); + + // We now have the full witness for our circuit + let pos: u64 = merkle_path.position().into(); + Some(Spend { + value_commitment_opening: Some(value_commitment_opening), + proof_generation_key: Some(proof_generation_key), + payment_address: Some(payment_address), + commitment_randomness: Some(note.rcm()), + ar: Some(alpha), + auth_path: merkle_path + .path_elems() + .iter() + .enumerate() + .map(|(i, node)| Some(((*node).into(), pos >> i & 0x1 == 1))) + .collect(), + anchor: Some(anchor), + }) + } + + fn create_proof(&self, circuit: Spend, rng: &mut R) -> Self::Proof { + create_random_proof(circuit, &self.0, rng).expect("proving should not fail") + } +} + /// A context object for creating the Sapling components of a Zcash transaction. pub struct SaplingProvingContext { bsk: TrapdoorSum, @@ -62,44 +113,26 @@ impl SaplingProvingContext { self.bsk += &rcv; // Construct the value commitment - let value_commitment_opening = ValueCommitmentOpening { - value, - randomness: rcv.inner(), - }; - let value_commitment = ValueCommitment::derive(NoteValue::from_raw(value), rcv); - - // Construct the viewing key - let viewing_key = proof_generation_key.to_viewing_key(); - - // Construct the payment address with the viewing key / diversifier - let payment_address = viewing_key.to_payment_address(diversifier).ok_or(())?; + let value = NoteValue::from_raw(value); + let value_commitment = ValueCommitment::derive(value, rcv.clone()); // This is the result of the re-randomization, we compute it for the caller let rk = PublicKey(proof_generation_key.ak.into()).randomize(ar, SPENDING_KEY_GENERATOR); - // Let's compute the nullifier while we have the position - let note = Note::from_parts(payment_address, NoteValue::from_raw(value), rseed); - - // We now have the full witness for our circuit - let pos: u64 = merkle_path.position().into(); - let instance = Spend { - value_commitment_opening: Some(value_commitment_opening), - proof_generation_key: Some(proof_generation_key), - payment_address: Some(payment_address), - commitment_randomness: Some(note.rcm()), - ar: Some(ar), - auth_path: merkle_path - .path_elems() - .iter() - .enumerate() - .map(|(i, node)| Some(((*node).into(), pos >> i & 0x1 == 1))) - .collect(), - anchor: Some(anchor), - }; + let instance = SpendParameters::prepare_circuit( + proof_generation_key, + diversifier, + rseed, + value, + ar, + rcv, + anchor, + merkle_path, + ) + .ok_or(())?; // Create proof - let proof = create_random_proof(instance, &proving_key.0, &mut rng) - .expect("proving should not fail"); + let proof = proving_key.create_proof(instance, &mut rng); // Accumulate the value commitment in the context self.cv_sum += &value_commitment; From 0d46fe72cc9e81154e78f41b6936f58a5fd6ccd2 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 2 Oct 2023 14:29:07 +0000 Subject: [PATCH 0219/1122] zcash_primitives: Introduce an `OutputProver` trait --- zcash_primitives/CHANGELOG.md | 2 +- zcash_primitives/src/sapling/prover.rs | 26 ++++++++++++++ zcash_proofs/src/sapling/prover.rs | 49 ++++++++++++++++++-------- 3 files changed, 62 insertions(+), 15 deletions(-) diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 756d67ef37..5db0809c42 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -11,7 +11,7 @@ and this library adheres to Rust's notion of - `zcash_primitives::sapling`: - `circuit` module (moved from `zcash_proofs::circuit::sapling`). - `constants` module. - - `prover::SpendProver` + - `prover::{SpendProver, OutputProver}` ### Removed - `zcash_primitives::constants`: diff --git a/zcash_primitives/src/sapling/prover.rs b/zcash_primitives/src/sapling/prover.rs index dcf0d65058..4f1d32ebee 100644 --- a/zcash_primitives/src/sapling/prover.rs +++ b/zcash_primitives/src/sapling/prover.rs @@ -44,6 +44,32 @@ pub trait SpendProver { ) -> Self::Proof; } +/// Interface for creating Sapling Output proofs. +pub trait OutputProver { + /// The proof type created by this prover. + type Proof; + + /// Prepares an instance of the Sapling Output circuit for the given inputs. + /// + /// Returns `None` if `diversifier` is not a valid Sapling diversifier. + fn prepare_circuit( + esk: jubjub::Fr, + payment_address: PaymentAddress, + rcm: jubjub::Fr, + value: NoteValue, + rcv: ValueCommitTrapdoor, + ) -> sapling::circuit::Output; + + /// Create the proof for a Sapling [`OutputDescription`]. + /// + /// [`OutputDescription`]: crate::transaction::components::OutputDescription + fn create_proof( + &self, + circuit: sapling::circuit::Output, + rng: &mut R, + ) -> Self::Proof; +} + /// Interface for creating zero-knowledge proofs for shielded transactions. pub trait TxProver { /// Type for persisting any necessary context across multiple Sapling proofs. diff --git a/zcash_proofs/src/sapling/prover.rs b/zcash_proofs/src/sapling/prover.rs index 6398f61190..faac73e99b 100644 --- a/zcash_proofs/src/sapling/prover.rs +++ b/zcash_proofs/src/sapling/prover.rs @@ -6,7 +6,7 @@ use zcash_primitives::{ sapling::{ circuit::{Output, Spend, ValueCommitmentOpening}, constants::{SPENDING_KEY_GENERATOR, VALUE_COMMITMENT_RANDOMNESS_GENERATOR}, - prover::SpendProver, + prover::{OutputProver, SpendProver}, redjubjub::{PublicKey, Signature}, value::{CommitmentSum, NoteValue, TrapdoorSum, ValueCommitTrapdoor, ValueCommitment}, Diversifier, MerklePath, Note, PaymentAddress, ProofGenerationKey, Rseed, @@ -66,6 +66,36 @@ impl SpendProver for SpendParameters { } } +impl OutputProver for OutputParameters { + type Proof = Proof; + + fn prepare_circuit( + esk: jubjub::Fr, + payment_address: PaymentAddress, + rcm: jubjub::Fr, + value: NoteValue, + rcv: ValueCommitTrapdoor, + ) -> Output { + // Construct the value commitment for the proof instance + let value_commitment_opening = ValueCommitmentOpening { + value: value.inner(), + randomness: rcv.inner(), + }; + + // We now have a full witness for the output proof. + Output { + value_commitment_opening: Some(value_commitment_opening), + payment_address: Some(payment_address), + commitment_randomness: Some(rcm), + esk: Some(esk), + } + } + + fn create_proof(&self, circuit: Output, rng: &mut R) -> Self::Proof { + create_random_proof(circuit, &self.0, rng).expect("proving should not fail") + } +} + /// A context object for creating the Sapling components of a Zcash transaction. pub struct SaplingProvingContext { bsk: TrapdoorSum, @@ -163,23 +193,14 @@ impl SaplingProvingContext { self.bsk -= &rcv; // Outputs subtract from the total. // Construct the value commitment for the proof instance - let value_commitment_opening = ValueCommitmentOpening { - value, - randomness: rcv.inner(), - }; - let value_commitment = ValueCommitment::derive(NoteValue::from_raw(value), rcv); + let value = NoteValue::from_raw(value); + let value_commitment = ValueCommitment::derive(value, rcv.clone()); // We now have a full witness for the output proof. - let instance = Output { - value_commitment_opening: Some(value_commitment_opening), - payment_address: Some(payment_address), - commitment_randomness: Some(rcm), - esk: Some(esk), - }; + let instance = OutputParameters::prepare_circuit(esk, payment_address, rcm, value, rcv); // Create proof - let proof = create_random_proof(instance, &proving_key.0, &mut rng) - .expect("proving should not fail"); + let proof = proving_key.create_proof(instance, &mut rng); // Accumulate the value commitment in the context. We do this to check internal consistency. self.cv_sum -= &value_commitment; // Outputs subtract from the total. From b9f8ec8ed28454fb18a130115c6818c3f068a450 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 3 Oct 2023 09:18:52 -0600 Subject: [PATCH 0220/1122] add pub getter to transaction builder use getter in original context to DRY add doc-comment Minor cleanup & documentation fixes. Co-Authored-By: Daira Emma Hopwood --- zcash_primitives/src/transaction/builder.rs | 28 +++++++++++++-------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/zcash_primitives/src/transaction/builder.rs b/zcash_primitives/src/transaction/builder.rs index ae016adc05..d19d438f81 100644 --- a/zcash_primitives/src/transaction/builder.rs +++ b/zcash_primitives/src/transaction/builder.rs @@ -403,16 +403,12 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { .ok_or(BalanceError::Overflow) } - /// Builds a transaction from the configured spends and outputs. + /// Reports the calculated fee given the specified fee rule. /// - /// Upon success, returns a tuple containing the final transaction, and the - /// [`SaplingMetadata`] generated during the build process. - pub fn build( - self, - prover: &impl TxProver, - fee_rule: &FR, - ) -> Result<(Transaction, SaplingMetadata), Error> { - let fee = fee_rule + /// This fee is a function of the spends and outputs that have been added to the builder, + /// pursuant to the specified [`FeeRule`]. + pub fn get_fee(&self, fee_rule: &FR) -> Result> { + fee_rule .fee_required( &self.params, self.target_height, @@ -432,7 +428,19 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { n => n, }, ) - .map_err(Error::Fee)?; + .map_err(Error::Fee) + } + + /// Builds a transaction from the configured spends and outputs. + /// + /// Upon success, returns a tuple containing the final transaction, and the + /// [`SaplingMetadata`] generated during the build process. + pub fn build( + self, + prover: &impl TxProver, + fee_rule: &FR, + ) -> Result<(Transaction, SaplingMetadata), Error> { + let fee = self.get_fee(fee_rule)?; self.build_internal(prover, fee) } From 296f75954bc693d372cd6fd1bd8cccbb9fa6c5c3 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 6 Oct 2023 20:19:41 +0000 Subject: [PATCH 0221/1122] Add `SpendProver::encode_proof` and `OutputProver::encode_proof` --- zcash_primitives/src/sapling/prover.rs | 12 +++++++++++- zcash_proofs/src/sapling/prover.rs | 18 +++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/zcash_primitives/src/sapling/prover.rs b/zcash_primitives/src/sapling/prover.rs index 4f1d32ebee..50af2e1db5 100644 --- a/zcash_primitives/src/sapling/prover.rs +++ b/zcash_primitives/src/sapling/prover.rs @@ -9,7 +9,7 @@ use crate::{ value::{NoteValue, ValueCommitTrapdoor, ValueCommitment}, MerklePath, }, - transaction::components::{Amount, GROTH_PROOF_SIZE}, + transaction::components::{sapling::GrothProofBytes, Amount, GROTH_PROOF_SIZE}, }; use super::{Diversifier, PaymentAddress, ProofGenerationKey, Rseed}; @@ -42,6 +42,11 @@ pub trait SpendProver { circuit: sapling::circuit::Spend, rng: &mut R, ) -> Self::Proof; + + /// Encodes the given Sapling [`SpendDescription`] proof, erasing its type. + /// + /// [`SpendDescription`]: crate::transaction::components::SpendDescription + fn encode_proof(proof: Self::Proof) -> GrothProofBytes; } /// Interface for creating Sapling Output proofs. @@ -68,6 +73,11 @@ pub trait OutputProver { circuit: sapling::circuit::Output, rng: &mut R, ) -> Self::Proof; + + /// Encodes the given Sapling [`OutputDescription`] proof, erasing its type. + /// + /// [`OutputDescription`]: crate::transaction::components::OutputDescription + fn encode_proof(proof: Self::Proof) -> GrothProofBytes; } /// Interface for creating zero-knowledge proofs for shielded transactions. diff --git a/zcash_proofs/src/sapling/prover.rs b/zcash_proofs/src/sapling/prover.rs index faac73e99b..7255783c7c 100644 --- a/zcash_proofs/src/sapling/prover.rs +++ b/zcash_proofs/src/sapling/prover.rs @@ -11,7 +11,7 @@ use zcash_primitives::{ value::{CommitmentSum, NoteValue, TrapdoorSum, ValueCommitTrapdoor, ValueCommitment}, Diversifier, MerklePath, Note, PaymentAddress, ProofGenerationKey, Rseed, }, - transaction::components::Amount, + transaction::components::{sapling::GrothProofBytes, Amount, GROTH_PROOF_SIZE}, }; use crate::{OutputParameters, SpendParameters}; @@ -64,6 +64,14 @@ impl SpendProver for SpendParameters { fn create_proof(&self, circuit: Spend, rng: &mut R) -> Self::Proof { create_random_proof(circuit, &self.0, rng).expect("proving should not fail") } + + fn encode_proof(proof: Self::Proof) -> GrothProofBytes { + let mut zkproof = [0u8; GROTH_PROOF_SIZE]; + proof + .write(&mut zkproof[..]) + .expect("should be able to serialize a proof"); + zkproof + } } impl OutputProver for OutputParameters { @@ -94,6 +102,14 @@ impl OutputProver for OutputParameters { fn create_proof(&self, circuit: Output, rng: &mut R) -> Self::Proof { create_random_proof(circuit, &self.0, rng).expect("proving should not fail") } + + fn encode_proof(proof: Self::Proof) -> GrothProofBytes { + let mut zkproof = [0u8; GROTH_PROOF_SIZE]; + proof + .write(&mut zkproof[..]) + .expect("should be able to serialize a proof"); + zkproof + } } /// A context object for creating the Sapling components of a Zcash transaction. From 46903fabf5061403baecf8fb88942f1482690e5a Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 6 Oct 2023 20:25:07 +0000 Subject: [PATCH 0222/1122] zcash_proofs: `impl {SpendProver, OutputProver} for LocalTxProver` --- zcash_proofs/CHANGELOG.md | 4 +- zcash_proofs/src/prover.rs | 81 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/zcash_proofs/CHANGELOG.md b/zcash_proofs/CHANGELOG.md index f544343b62..1971f93597 100644 --- a/zcash_proofs/CHANGELOG.md +++ b/zcash_proofs/CHANGELOG.md @@ -8,6 +8,8 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added - `zcash_proofs::{SpendParameters, OutputParameters}` +- `impl zcash_primitives::sapling::prover::{SpendProver, OutputProver}` for + `zcash_proofs::prover::LocalTxProver` ### Changed - The new `SpendParameters` and `OutputParameters` types are used in the @@ -110,7 +112,7 @@ and this library adheres to Rust's notion of ### Added - `zcash_proofs::ZcashParameters` - `zcash_proofs::parse_parameters` -- `zcash_proofs::prover::LocalProver::from_bytes` +- `zcash_proofs::prover::LocalTxProver::from_bytes` - The `zcash_proofs::constants` module, containing constants and helpers used by the `zcash_proofs::circuit::ecc::fixed_base_multiplication` gadget: - The `FixedGeneratorOwned` type alias. diff --git a/zcash_proofs/src/prover.rs b/zcash_proofs/src/prover.rs index 8bbd1abff6..3ddbcfcd4a 100644 --- a/zcash_proofs/src/prover.rs +++ b/zcash_proofs/src/prover.rs @@ -1,16 +1,17 @@ //! Abstractions over the proving system and parameters for ease of use. -use bellman::groth16::PreparedVerifyingKey; +use bellman::groth16::{PreparedVerifyingKey, Proof}; use bls12_381::Bls12; use std::path::Path; use zcash_primitives::{ sapling::{ - prover::TxProver, + self, + prover::{OutputProver, SpendProver, TxProver}, redjubjub::{PublicKey, Signature}, - value::ValueCommitment, + value::{NoteValue, ValueCommitTrapdoor, ValueCommitment}, Diversifier, MerklePath, PaymentAddress, ProofGenerationKey, Rseed, }, - transaction::components::{Amount, GROTH_PROOF_SIZE}, + transaction::components::{sapling::GrothProofBytes, Amount, GROTH_PROOF_SIZE}, }; use crate::{ @@ -143,6 +144,78 @@ impl LocalTxProver { } } +impl SpendProver for LocalTxProver { + type Proof = Proof; + + fn prepare_circuit( + proof_generation_key: ProofGenerationKey, + diversifier: Diversifier, + rseed: Rseed, + value: NoteValue, + alpha: jubjub::Fr, + rcv: ValueCommitTrapdoor, + anchor: bls12_381::Scalar, + merkle_path: MerklePath, + ) -> Option { + SpendParameters::prepare_circuit( + proof_generation_key, + diversifier, + rseed, + value, + alpha, + rcv, + anchor, + merkle_path, + ) + } + + fn create_proof( + &self, + circuit: sapling::circuit::Spend, + rng: &mut R, + ) -> Self::Proof { + self.spend_params.create_proof(circuit, rng) + } + + fn encode_proof(proof: Self::Proof) -> GrothProofBytes { + let mut zkproof = [0u8; GROTH_PROOF_SIZE]; + proof + .write(&mut zkproof[..]) + .expect("should be able to serialize a proof"); + zkproof + } +} + +impl OutputProver for LocalTxProver { + type Proof = Proof; + + fn prepare_circuit( + esk: jubjub::Fr, + payment_address: PaymentAddress, + rcm: jubjub::Fr, + value: NoteValue, + rcv: ValueCommitTrapdoor, + ) -> sapling::circuit::Output { + OutputParameters::prepare_circuit(esk, payment_address, rcm, value, rcv) + } + + fn create_proof( + &self, + circuit: sapling::circuit::Output, + rng: &mut R, + ) -> Self::Proof { + self.output_params.create_proof(circuit, rng) + } + + fn encode_proof(proof: Self::Proof) -> GrothProofBytes { + let mut zkproof = [0u8; GROTH_PROOF_SIZE]; + proof + .write(&mut zkproof[..]) + .expect("should be able to serialize a proof"); + zkproof + } +} + impl TxProver for LocalTxProver { type SaplingProvingContext = SaplingProvingContext; From 04aa5a044b7c3623b62f0b5642b2b3ff4b6216bd Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 6 Oct 2023 20:31:31 +0000 Subject: [PATCH 0223/1122] zcash_primitives: Add `MockSpendProver` and `MockOutputProver` These are analogues of `MockTxProver` in that they implement the corresponding Sapling prover traits. --- zcash_primitives/CHANGELOG.md | 2 + zcash_primitives/src/sapling/prover.rs | 88 +++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 5db0809c42..0cc06dc311 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -12,6 +12,8 @@ and this library adheres to Rust's notion of - `circuit` module (moved from `zcash_proofs::circuit::sapling`). - `constants` module. - `prover::{SpendProver, OutputProver}` +- Test helpers, behind the `test-dependencies` feature flag: + - `zcash_primitives::prover::mock::{MockSpendProver, MockOutputProver}` ### Removed - `zcash_primitives::constants`: diff --git a/zcash_primitives/src/sapling/prover.rs b/zcash_primitives/src/sapling/prover.rs index 50af2e1db5..e047d2dccc 100644 --- a/zcash_primitives/src/sapling/prover.rs +++ b/zcash_primitives/src/sapling/prover.rs @@ -133,20 +133,104 @@ pub trait TxProver { #[cfg(any(test, feature = "test-dependencies"))] pub mod mock { + use ff::Field; use rand_core::OsRng; - use super::TxProver; + use super::{OutputProver, SpendProver, TxProver}; use crate::{ sapling::{ self, + circuit::ValueCommitmentOpening, constants::SPENDING_KEY_GENERATOR, redjubjub::{PublicKey, Signature}, value::{NoteValue, ValueCommitTrapdoor, ValueCommitment}, Diversifier, PaymentAddress, ProofGenerationKey, Rseed, }, - transaction::components::{Amount, GROTH_PROOF_SIZE}, + transaction::components::{sapling::GrothProofBytes, Amount, GROTH_PROOF_SIZE}, }; + pub struct MockSpendProver; + + impl SpendProver for MockSpendProver { + type Proof = GrothProofBytes; + + fn prepare_circuit( + proof_generation_key: ProofGenerationKey, + diversifier: Diversifier, + _rseed: Rseed, + value: NoteValue, + alpha: jubjub::Fr, + rcv: ValueCommitTrapdoor, + anchor: bls12_381::Scalar, + _merkle_path: sapling::MerklePath, + ) -> Option { + let payment_address = proof_generation_key + .to_viewing_key() + .ivk() + .to_payment_address(diversifier); + Some(sapling::circuit::Spend { + value_commitment_opening: Some(ValueCommitmentOpening { + value: value.inner(), + randomness: rcv.inner(), + }), + proof_generation_key: Some(proof_generation_key), + payment_address, + commitment_randomness: Some(jubjub::Scalar::ZERO), + ar: Some(alpha), + auth_path: vec![], + anchor: Some(anchor), + }) + } + + fn create_proof( + &self, + _circuit: sapling::circuit::Spend, + _rng: &mut R, + ) -> Self::Proof { + [0u8; GROTH_PROOF_SIZE] + } + + fn encode_proof(proof: Self::Proof) -> GrothProofBytes { + proof + } + } + + pub struct MockOutputProver; + + impl OutputProver for MockOutputProver { + type Proof = GrothProofBytes; + + fn prepare_circuit( + esk: jubjub::Fr, + payment_address: PaymentAddress, + rcm: jubjub::Fr, + value: NoteValue, + rcv: ValueCommitTrapdoor, + ) -> sapling::circuit::Output { + sapling::circuit::Output { + value_commitment_opening: Some(ValueCommitmentOpening { + value: value.inner(), + randomness: rcv.inner(), + }), + payment_address: Some(payment_address), + commitment_randomness: Some(rcm), + esk: Some(esk), + } + } + + fn create_proof( + &self, + _circuit: sapling::circuit::Output, + _rng: &mut R, + ) -> Self::Proof { + [0u8; GROTH_PROOF_SIZE] + } + + fn encode_proof(proof: Self::Proof) -> GrothProofBytes { + proof + } + } + pub struct MockTxProver; impl TxProver for MockTxProver { From 241a1a36607a3b1eb7b8129425d7745ba98cf336 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 6 Oct 2023 20:45:19 +0000 Subject: [PATCH 0224/1122] zcash_primitives: Add some more `Clone` and `Debug` impls --- zcash_primitives/CHANGELOG.md | 1 + zcash_primitives/src/sapling/circuit.rs | 18 ++++++++++++++++++ zcash_primitives/src/sapling/keys.rs | 23 ++++++++++++++++------- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 0cc06dc311..f8a26f20ea 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -12,6 +12,7 @@ and this library adheres to Rust's notion of - `circuit` module (moved from `zcash_proofs::circuit::sapling`). - `constants` module. - `prover::{SpendProver, OutputProver}` + - `impl Debug for keys::{ExpandedSpendingKey, ProofGenerationKey}` - Test helpers, behind the `test-dependencies` feature flag: - `zcash_primitives::prover::mock::{MockSpendProver, MockOutputProver}` diff --git a/zcash_primitives/src/sapling/circuit.rs b/zcash_primitives/src/sapling/circuit.rs index 78ef9b8472..036941f225 100644 --- a/zcash_primitives/src/sapling/circuit.rs +++ b/zcash_primitives/src/sapling/circuit.rs @@ -1,5 +1,7 @@ //! The Sapling circuits. +use core::fmt; + use group::{ff::PrimeField, Curve}; use bellman::{Circuit, ConstraintSystem, SynthesisError}; @@ -46,6 +48,7 @@ impl ValueCommitmentOpening { } /// This is an instance of the `Spend` circuit. +#[derive(Clone)] pub struct Spend { /// The opening of a Pedersen commitment to the value being spent. pub value_commitment_opening: Option, @@ -71,7 +74,16 @@ pub struct Spend { pub anchor: Option, } +impl fmt::Debug for Spend { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Spend") + .field("anchor", &self.anchor) + .finish_non_exhaustive() + } +} + /// This is an output circuit instance. +#[derive(Clone)] pub struct Output { /// The opening of a Pedersen commitment to the value being spent. pub value_commitment_opening: Option, @@ -86,6 +98,12 @@ pub struct Output { pub esk: Option, } +impl fmt::Debug for Output { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Output").finish_non_exhaustive() + } +} + /// Exposes a Pedersen commitment to the value as an /// input to the circuit fn expose_value_commitment( diff --git a/zcash_primitives/src/sapling/keys.rs b/zcash_primitives/src/sapling/keys.rs index 3fd458f490..aa26edd6ae 100644 --- a/zcash_primitives/src/sapling/keys.rs +++ b/zcash_primitives/src/sapling/keys.rs @@ -4,6 +4,7 @@ //! //! [section 4.2.2]: https://zips.z.cash/protocol/protocol.pdf#saplingkeycomponents +use std::fmt; use std::io::{self, Read, Write}; use super::{ @@ -46,6 +47,13 @@ pub struct ExpandedSpendingKey { pub ovk: OutgoingViewingKey, } +impl fmt::Debug for ExpandedSpendingKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExpandedSpendingKey") + .finish_non_exhaustive() + } +} + impl ExpandedSpendingKey { pub fn from_spending_key(sk: &[u8]) -> Self { let ask = jubjub::Fr::from_bytes_wide(prf_expand(sk, &[0x00]).as_array()); @@ -119,6 +127,14 @@ pub struct ProofGenerationKey { pub nsk: jubjub::Fr, } +impl fmt::Debug for ProofGenerationKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProofGenerationKey") + .field("ak", &self.ak) + .finish_non_exhaustive() + } +} + impl ProofGenerationKey { pub fn to_viewing_key(&self) -> ViewingKey { ViewingKey { @@ -473,16 +489,9 @@ impl SharedSecret { pub mod testing { use proptest::collection::vec; use proptest::prelude::*; - use std::fmt::{self, Debug, Formatter}; use super::{ExpandedSpendingKey, FullViewingKey, SaplingIvk}; - impl Debug for ExpandedSpendingKey { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "Spending keys cannot be Debug-formatted.") - } - } - prop_compose! { pub fn arb_expanded_spending_key()(v in vec(any::(), 32..252)) -> ExpandedSpendingKey { ExpandedSpendingKey::from_spending_key(&v) From b09b435135cff7f10c1a9d64b8577fc32c8dd255 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 6 Oct 2023 23:12:47 +0000 Subject: [PATCH 0225/1122] zcash_primitives: Change `sapling::MapAuth` to take `&mut self` --- zcash_primitives/CHANGELOG.md | 4 ++++ .../src/transaction/components/sapling.rs | 22 +++++++++++-------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index f8a26f20ea..a4367d97e0 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -16,6 +16,10 @@ and this library adheres to Rust's notion of - Test helpers, behind the `test-dependencies` feature flag: - `zcash_primitives::prover::mock::{MockSpendProver, MockOutputProver}` +### Changed +- `zcash_primitives::transaction::components::sapling`: + - `MapAuth` trait methods now take `&mut self` instead of `&self`. + ### Removed - `zcash_primitives::constants`: - All `const` values (moved to `zcash_primitives::sapling::constants`). diff --git a/zcash_primitives/src/transaction/components/sapling.rs b/zcash_primitives/src/transaction/components/sapling.rs index f899155e8c..fc275f1ed0 100644 --- a/zcash_primitives/src/transaction/components/sapling.rs +++ b/zcash_primitives/src/transaction/components/sapling.rs @@ -56,11 +56,14 @@ impl Authorization for Authorized { type AuthSig = redjubjub::Signature; } +/// A map from one bundle authorization to another. +/// +/// For use with [`Bundle::map_authorization`]. pub trait MapAuth { - fn map_spend_proof(&self, p: A::SpendProof) -> B::SpendProof; - fn map_output_proof(&self, p: A::OutputProof) -> B::OutputProof; - fn map_auth_sig(&self, s: A::AuthSig) -> B::AuthSig; - fn map_authorization(&self, a: A) -> B; + fn map_spend_proof(&mut self, p: A::SpendProof) -> B::SpendProof; + fn map_output_proof(&mut self, p: A::OutputProof) -> B::OutputProof; + fn map_auth_sig(&mut self, s: A::AuthSig) -> B::AuthSig; + fn map_authorization(&mut self, a: A) -> B; } /// The identity map. @@ -71,27 +74,27 @@ pub trait MapAuth { /// [`TransactionData::map_authorization`]: crate::transaction::TransactionData::map_authorization impl MapAuth for () { fn map_spend_proof( - &self, + &mut self, p: ::SpendProof, ) -> ::SpendProof { p } fn map_output_proof( - &self, + &mut self, p: ::OutputProof, ) -> ::OutputProof { p } fn map_auth_sig( - &self, + &mut self, s: ::AuthSig, ) -> ::AuthSig { s } - fn map_authorization(&self, a: Authorized) -> Authorized { + fn map_authorization(&mut self, a: Authorized) -> Authorized { a } } @@ -160,7 +163,8 @@ impl Bundle { &self.authorization } - pub fn map_authorization>(self, f: F) -> Bundle { + /// Transitions this bundle from one authorization state to another. + pub fn map_authorization>(self, mut f: F) -> Bundle { Bundle { shielded_spends: self .shielded_spends From 65efee1a16e169b34b411e039782196d835c9e30 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 6 Oct 2023 23:16:59 +0000 Subject: [PATCH 0226/1122] zcash_primitives: Add `sapling::Bundle::try_map_authorization` --- zcash_primitives/CHANGELOG.md | 3 ++ .../src/transaction/components/sapling.rs | 50 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index a4367d97e0..33895ad824 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -13,6 +13,9 @@ and this library adheres to Rust's notion of - `constants` module. - `prover::{SpendProver, OutputProver}` - `impl Debug for keys::{ExpandedSpendingKey, ProofGenerationKey}` +- `zcash_primitives::transaction::components::sapling`: + - `Bundle::try_map_authorization` + - `TryMapAuth` - Test helpers, behind the `test-dependencies` feature flag: - `zcash_primitives::prover::mock::{MockSpendProver, MockOutputProver}` diff --git a/zcash_primitives/src/transaction/components/sapling.rs b/zcash_primitives/src/transaction/components/sapling.rs index fc275f1ed0..bfd4d24929 100644 --- a/zcash_primitives/src/transaction/components/sapling.rs +++ b/zcash_primitives/src/transaction/components/sapling.rs @@ -99,6 +99,17 @@ impl MapAuth for () { } } +/// A fallible map from one bundle authorization to another. +/// +/// For use with [`Bundle::try_map_authorization`]. +pub trait TryMapAuth { + type Error; + fn try_map_spend_proof(&mut self, p: A::SpendProof) -> Result; + fn try_map_output_proof(&mut self, p: A::OutputProof) -> Result; + fn try_map_auth_sig(&mut self, s: A::AuthSig) -> Result; + fn try_map_authorization(&mut self, a: A) -> Result; +} + #[derive(Debug, Clone)] pub struct Bundle { shielded_spends: Vec>, @@ -194,6 +205,45 @@ impl Bundle { authorization: f.map_authorization(self.authorization), } } + + /// Transitions this bundle from one authorization state to another. + pub fn try_map_authorization>( + self, + mut f: F, + ) -> Result, F::Error> { + Ok(Bundle { + shielded_spends: self + .shielded_spends + .into_iter() + .map(|d| { + Ok(SpendDescription { + cv: d.cv, + anchor: d.anchor, + nullifier: d.nullifier, + rk: d.rk, + zkproof: f.try_map_spend_proof(d.zkproof)?, + spend_auth_sig: f.try_map_auth_sig(d.spend_auth_sig)?, + }) + }) + .collect::>()?, + shielded_outputs: self + .shielded_outputs + .into_iter() + .map(|o| { + Ok(OutputDescription { + cv: o.cv, + cmu: o.cmu, + ephemeral_key: o.ephemeral_key, + enc_ciphertext: o.enc_ciphertext, + out_ciphertext: o.out_ciphertext, + zkproof: f.try_map_output_proof(o.zkproof)?, + }) + }) + .collect::>()?, + value_balance: self.value_balance, + authorization: f.try_map_authorization(self.authorization)?, + }) + } } impl DynamicUsage for Bundle { From e1a4238a71474bb3bb23b081e3443f670d7550f0 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 6 Oct 2023 23:40:32 +0000 Subject: [PATCH 0227/1122] zcash_primitives: Add helper impls of `{Try}MapAuth` for closures --- zcash_primitives/CHANGELOG.md | 3 + .../src/transaction/components/sapling.rs | 56 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 33895ad824..20bb1e4d26 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -16,6 +16,9 @@ and this library adheres to Rust's notion of - `zcash_primitives::transaction::components::sapling`: - `Bundle::try_map_authorization` - `TryMapAuth` + - `impl {MapAuth, TryMapAuth} for (FnMut, FnMut, FnMut, FnMut)` helpers to + enable calling `Bundle::{map_authorization, try_map_authorization}` with a + set of closures. - Test helpers, behind the `test-dependencies` feature flag: - `zcash_primitives::prover::mock::{MockSpendProver, MockOutputProver}` diff --git a/zcash_primitives/src/transaction/components/sapling.rs b/zcash_primitives/src/transaction/components/sapling.rs index bfd4d24929..1c17eaf364 100644 --- a/zcash_primitives/src/transaction/components/sapling.rs +++ b/zcash_primitives/src/transaction/components/sapling.rs @@ -99,6 +99,33 @@ impl MapAuth for () { } } +/// A helper for implementing `MapAuth` with a set of closures. +impl MapAuth for (F, G, H, I) +where + A: Authorization, + B: Authorization, + F: FnMut(A::SpendProof) -> B::SpendProof, + G: FnMut(A::OutputProof) -> B::OutputProof, + H: FnMut(A::AuthSig) -> B::AuthSig, + I: FnMut(A) -> B, +{ + fn map_spend_proof(&mut self, p: A::SpendProof) -> B::SpendProof { + self.0(p) + } + + fn map_output_proof(&mut self, p: A::OutputProof) -> B::OutputProof { + self.1(p) + } + + fn map_auth_sig(&mut self, s: A::AuthSig) -> B::AuthSig { + self.2(s) + } + + fn map_authorization(&mut self, a: A) -> B { + self.3(a) + } +} + /// A fallible map from one bundle authorization to another. /// /// For use with [`Bundle::try_map_authorization`]. @@ -110,6 +137,35 @@ pub trait TryMapAuth { fn try_map_authorization(&mut self, a: A) -> Result; } +/// A helper for implementing `TryMapAuth` with a set of closures. +impl TryMapAuth for (F, G, H, I) +where + A: Authorization, + B: Authorization, + F: FnMut(A::SpendProof) -> Result, + G: FnMut(A::OutputProof) -> Result, + H: FnMut(A::AuthSig) -> Result, + I: FnMut(A) -> Result, +{ + type Error = E; + + fn try_map_spend_proof(&mut self, p: A::SpendProof) -> Result { + self.0(p) + } + + fn try_map_output_proof(&mut self, p: A::OutputProof) -> Result { + self.1(p) + } + + fn try_map_auth_sig(&mut self, s: A::AuthSig) -> Result { + self.2(s) + } + + fn try_map_authorization(&mut self, a: A) -> Result { + self.3(a) + } +} + #[derive(Debug, Clone)] pub struct Bundle { shielded_spends: Vec>, From a9d18ec2ce64905202f990e0191693845d07da4f Mon Sep 17 00:00:00 2001 From: Matthew Watt Date: Sat, 7 Oct 2023 15:30:07 -0500 Subject: [PATCH 0228/1122] Gracefully handle attempted spend to a UA with neither transparent nor sapling recipients --- zcash_client_backend/src/data_api/error.rs | 4 ++++ zcash_client_backend/src/data_api/wallet.rs | 26 ++++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index 31a9b39c99..f5297116b7 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -54,6 +54,9 @@ pub enum Error write!(f, "Must scan blocks first"), Error::Builder(e) => write!(f, "An error occurred building the transaction: {}", e), Error::MemoForbidden => write!(f, "It is not possible to send a memo to a transparent address."), + Error::UnsupportedOutputType => write!(f, "Attempted to create spend to an unsupported output type"), Error::NoteMismatch(n) => write!(f, "A note being spent ({}) does not correspond to either the internal or external full viewing key for the provided spending key.", n), #[cfg(feature = "transparent-inputs")] diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 0d1adda6fe..61da029834 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -546,12 +546,26 @@ where .memo .as_ref() .map_or_else(MemoBytes::empty, |m| m.clone()); - builder.add_sapling_output( - external_ovk, - *ua.sapling().expect("TODO: Add Orchard support to builder"), - payment.amount, - memo.clone(), - )?; + + if ua.sapling().is_some() { + builder.add_sapling_output( + external_ovk, + *ua.sapling().unwrap(), + payment.amount, + memo.clone(), + )?; + } else if ua.transparent().is_some() { + if payment.memo.is_some() { + return Err(Error::MemoForbidden); + } else { + builder.add_transparent_output( + ua.transparent().unwrap(), + payment.amount + )?; + } + } else { + return Err(Error::UnsupportedOutputType); + } sapling_output_meta.push(( Recipient::Unified(ua.clone(), PoolType::Shielded(ShieldedProtocol::Sapling)), payment.amount, From 94f2240e0890e9844f85fcdde0d7a6647b5bef8f Mon Sep 17 00:00:00 2001 From: Matthew Watt Date: Sun, 8 Oct 2023 19:05:50 -0500 Subject: [PATCH 0229/1122] Provide PoolType to UnsupportedPoolType case; stylistic changes --- zcash_client_backend/CHANGELOG.md | 5 ++++ zcash_client_backend/src/data_api.rs | 3 ++- zcash_client_backend/src/data_api/error.rs | 26 +++++++++++---------- zcash_client_backend/src/data_api/wallet.rs | 16 ++++++++----- zcash_client_sqlite/CHANGELOG.md | 4 ++++ zcash_client_sqlite/src/error.rs | 7 +++++- zcash_client_sqlite/src/wallet.rs | 9 ++++++- 7 files changed, 49 insertions(+), 21 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 449efeb5f3..0af57a371b 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -7,6 +7,11 @@ and this library adheres to Rust's notion of ## [Unreleased] +### Added +- `zcash_client_backend::data_api::ShieldedProtocol` has a new variant for `Orchard`, allowing for better reporting to callers trying to perform actions using `Orchard` before it is fully supported. +- `zcash_client_backend::data_api::error::Error` has new error variant: + - `Error::UnsupportedPoolType(zcash_client_backend::data_api::PoolType)` + ### Changed - `zcash_client_backend::data_api::chain::scan_cached_blocks` now returns a `ScanSummary` containing metadata about the scanned blocks on success. diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 875ac614b3..2fd0b5b2de 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -556,7 +556,8 @@ pub struct SentTransaction<'a> { pub enum ShieldedProtocol { /// The Sapling protocol Sapling, - // TODO: Orchard + /// The Orchard protocol + Orchard } /// A unique identifier for a shielded transaction output diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index f5297116b7..88357816fe 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -22,7 +22,7 @@ use zcash_primitives::{legacy::TransparentAddress, zip32::DiversifierIndex}; /// Errors that can occur as a consequence of wallet operations. #[derive(Debug)] -pub enum Error { +pub enum Error { /// An error occurred retrieving data from the underlying data source DataSource(DataSourceError), @@ -54,8 +54,8 @@ pub enum Error fmt::Display for Error +impl fmt::Display for Error where DE: fmt::Display, CE: fmt::Display, SE: fmt::Display, FE: fmt::Display, N: fmt::Display, + PT: fmt::Display { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match &self { @@ -114,7 +115,7 @@ where Error::ScanRequired => write!(f, "Must scan blocks first"), Error::Builder(e) => write!(f, "An error occurred building the transaction: {}", e), Error::MemoForbidden => write!(f, "It is not possible to send a memo to a transparent address."), - Error::UnsupportedOutputType => write!(f, "Attempted to create spend to an unsupported output type"), + Error::UnsupportedPoolType(t) => write!(f, "Attempted to create spend to an unsupported pool type: {}", t), Error::NoteMismatch(n) => write!(f, "A note being spent ({}) does not correspond to either the internal or external full viewing key for the provided spending key.", n), #[cfg(feature = "transparent-inputs")] @@ -133,13 +134,14 @@ where } } -impl error::Error for Error +impl error::Error for Error where DE: Debug + Display + error::Error + 'static, CE: Debug + Display + error::Error + 'static, SE: Debug + Display + error::Error + 'static, FE: Debug + Display + 'static, N: Debug + Display, + PT: Debug + Display { fn source(&self) -> Option<&(dyn error::Error + 'static)> { match &self { @@ -152,19 +154,19 @@ where } } -impl From> for Error { +impl From> for Error { fn from(e: builder::Error) -> Self { Error::Builder(e) } } -impl From for Error { +impl From for Error { fn from(e: BalanceError) -> Self { Error::BalanceError(e) } } -impl From> for Error { +impl From> for Error { fn from(e: InputSelectorError) -> Self { match e { InputSelectorError::DataSource(e) => Error::DataSource(e), @@ -181,19 +183,19 @@ impl From> for Error From for Error { +impl From for Error { fn from(e: sapling::builder::Error) -> Self { Error::Builder(builder::Error::SaplingBuild(e)) } } -impl From for Error { +impl From for Error { fn from(e: transparent::builder::Error) -> Self { Error::Builder(builder::Error::TransparentBuild(e)) } } -impl From> for Error { +impl From> for Error { fn from(e: ShardTreeError) -> Self { Error::CommitmentTree(e) } diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 61da029834..0e2568e3f3 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -207,6 +207,7 @@ pub fn create_spend_to_address( GreedyInputSelectorError, Infallible, DbT::NoteRef, + PoolType, >, > where @@ -310,6 +311,7 @@ pub fn spend( InputsT::Error, ::Error, DbT::NoteRef, + PoolType, >, > where @@ -364,6 +366,7 @@ pub fn propose_transfer( InputsT::Error, ::Error, DbT::NoteRef, + PoolType, >, > where @@ -444,6 +447,7 @@ pub fn create_proposed_transaction( InputsErrT, FeeRuleT::Error, DbT::NoteRef, + PoolType, >, > where @@ -489,7 +493,7 @@ where let checkpoint_depth = wallet_db.get_checkpoint_depth(min_confirmations)?; - wallet_db.with_sapling_tree_mut::<_, _, Error<_, _, _, _, _>>(|sapling_tree| { + wallet_db.with_sapling_tree_mut::<_, _, Error<_, _, _, _, _, _>>(|sapling_tree| { for selected in proposal.sapling_inputs() { let (note, key, merkle_path) = select_key_for_note( sapling_tree, @@ -547,24 +551,24 @@ where .as_ref() .map_or_else(MemoBytes::empty, |m| m.clone()); - if ua.sapling().is_some() { + if let Some(sapling_receiver) = ua.sapling() { builder.add_sapling_output( external_ovk, - *ua.sapling().unwrap(), + *sapling_receiver, payment.amount, memo.clone(), )?; - } else if ua.transparent().is_some() { + } else if let Some(taddr) = ua.transparent() { if payment.memo.is_some() { return Err(Error::MemoForbidden); } else { builder.add_transparent_output( - ua.transparent().unwrap(), + taddr, payment.amount )?; } } else { - return Err(Error::UnsupportedOutputType); + return Err(Error::UnsupportedPoolType(PoolType::Shielded(ShieldedProtocol::Orchard))); } sapling_output_meta.push(( Recipient::Unified(ua.clone(), PoolType::Shielded(ShieldedProtocol::Sapling)), diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 2a044fc2b2..f1419da2d3 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -7,6 +7,10 @@ and this library adheres to Rust's notion of ## [Unreleased] +### Added +- `zcash_client_sqlite::error::SqliteClientError` has new error variant: + - `SqliteClientError::UnsupportedPoolType(zcash_client_backend::data_api::PoolType)` + ## [0.8.0] - 2023-09-25 ### Notable Changes diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index b513fc3d12..177026e5e9 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -4,6 +4,7 @@ use std::error; use std::fmt; use shardtree::error::ShardTreeError; +use zcash_client_backend::data_api::PoolType; use zcash_client_backend::encoding::{Bech32DecodeError, TransparentCodecError}; use zcash_primitives::{consensus::BlockHeight, zip32::AccountId}; @@ -98,6 +99,9 @@ pub enum SqliteClientError { /// [`WalletWrite::update_chain_tip`]: /// zcash_client_backend::data_api::WalletWrite::update_chain_tip ChainHeightUnknown, + + /// Unsupported pool type + UnsupportedPoolType(PoolType) } impl error::Error for SqliteClientError { @@ -144,7 +148,8 @@ impl fmt::Display for SqliteClientError { SqliteClientError::AddressNotRecognized(_) => write!(f, "The address associated with a received txo is not identifiable as belonging to the wallet."), SqliteClientError::CommitmentTree(err) => write!(f, "An error occurred accessing or updating note commitment tree data: {}.", err), SqliteClientError::CacheMiss(height) => write!(f, "Requested height {} does not exist in the block cache.", height), - SqliteClientError::ChainHeightUnknown => write!(f, "Chain height unknown; please call `update_chain_tip`") + SqliteClientError::ChainHeightUnknown => write!(f, "Chain height unknown; please call `update_chain_tip`"), + SqliteClientError::UnsupportedPoolType(t) => write!(f, "Pool type is not currently supported: {:?}", t) } } } diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 715d30f7c5..41ea48e0f0 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -136,6 +136,7 @@ pub(crate) fn pool_code(pool_type: PoolType) -> i64 { match pool_type { PoolType::Transparent => 0i64, PoolType::Shielded(ShieldedProtocol::Sapling) => 2i64, + PoolType::Shielded(ShieldedProtocol::Orchard) => 4i64 } } @@ -758,7 +759,12 @@ pub(crate) fn get_received_memo( conn: &rusqlite::Connection, note_id: NoteId, ) -> Result, SqliteClientError> { - let memo_bytes: Option> = match note_id.protocol() { + let protocol = note_id.protocol(); + if let ShieldedProtocol::Orchard = protocol { + return Err(SqliteClientError::UnsupportedPoolType(PoolType::Shielded(protocol))); + } + + let memo_bytes: Option> = match protocol { ShieldedProtocol::Sapling => conn .query_row( "SELECT memo FROM sapling_received_notes @@ -773,6 +779,7 @@ pub(crate) fn get_received_memo( ) .optional()? .flatten(), + ShieldedProtocol::Orchard => None }; memo_bytes From 46ed4964c3612cab98516235b280e968d707ee5e Mon Sep 17 00:00:00 2001 From: Matthew Watt Date: Sun, 8 Oct 2023 19:19:51 -0500 Subject: [PATCH 0230/1122] Display for PoolType --- zcash_client_backend/src/data_api.rs | 12 +++++++++++- zcash_client_sqlite/src/error.rs | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 2fd0b5b2de..7be3321c5a 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -2,7 +2,7 @@ use std::{ collections::{BTreeMap, HashMap}, - fmt::Debug, + fmt::{self, Debug}, io, num::{NonZeroU32, TryFromIntError}, }; @@ -604,6 +604,16 @@ pub enum PoolType { Shielded(ShieldedProtocol), } +impl fmt::Display for PoolType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + PoolType::Transparent => f.write_str("Transparent"), + PoolType::Shielded(ShieldedProtocol::Sapling) => f.write_str("Sapling"), + PoolType::Shielded(ShieldedProtocol::Orchard) => f.write_str("Orchard"), + } + } +} + /// A type that represents the recipient of a transaction output; a recipient address (and, for /// unified addresses, the pool to which the payment is sent) in the case of outgoing output, or an /// internal account ID and the pool to which funds were sent in the case of a wallet-internal diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 177026e5e9..4d9977d762 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -149,7 +149,7 @@ impl fmt::Display for SqliteClientError { SqliteClientError::CommitmentTree(err) => write!(f, "An error occurred accessing or updating note commitment tree data: {}.", err), SqliteClientError::CacheMiss(height) => write!(f, "Requested height {} does not exist in the block cache.", height), SqliteClientError::ChainHeightUnknown => write!(f, "Chain height unknown; please call `update_chain_tip`"), - SqliteClientError::UnsupportedPoolType(t) => write!(f, "Pool type is not currently supported: {:?}", t) + SqliteClientError::UnsupportedPoolType(t) => write!(f, "Pool type is not currently supported: {}", t) } } } From 029b7f0d69f5b06e0fb8d799658107f1081b2a1d Mon Sep 17 00:00:00 2001 From: Matthew Watt Date: Sun, 8 Oct 2023 19:27:20 -0500 Subject: [PATCH 0231/1122] Document fallback behavior for Orchard-only UAs --- zcash_client_backend/src/data_api/wallet.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 0e2568e3f3..f58fd0be23 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -428,6 +428,9 @@ where /// /// Returns the database identifier for the newly constructed transaction, or an error if /// an error occurs in transaction construction, proving, or signing. +/// +/// Note: If the payment includes a recipient with an Orchard-only UA, this will attempt to fallback +/// to the transparent receiver until full Orchard support is implemented #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] pub fn create_proposed_transaction( From 9d7ac07b1ab7d1ad15a9e3160b4cebce98ff1a0a Mon Sep 17 00:00:00 2001 From: Matthew Watt Date: Sun, 8 Oct 2023 19:37:02 -0500 Subject: [PATCH 0232/1122] Fix test build failures --- zcash_client_sqlite/src/testing.rs | 6 +++++- zcash_client_sqlite/src/wallet/sapling.rs | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 386543c10c..0a2db28e78 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -15,7 +15,7 @@ use tempfile::NamedTempFile; use tempfile::TempDir; use zcash_client_backend::data_api::chain::ScanSummary; -use zcash_client_backend::data_api::{AccountBalance, WalletRead}; +use zcash_client_backend::data_api::{AccountBalance, WalletRead, PoolType}; #[allow(deprecated)] use zcash_client_backend::{ address::RecipientAddress, @@ -446,6 +446,7 @@ impl TestState { GreedyInputSelectorError, Infallible, ReceivedNoteId, + PoolType, >, > { let params = self.network(); @@ -479,6 +480,7 @@ impl TestState { InputsT::Error, ::Error, ReceivedNoteId, + PoolType, >, > where @@ -513,6 +515,7 @@ impl TestState { InputsT::Error, ::Error, ReceivedNoteId, + PoolType, >, > where @@ -579,6 +582,7 @@ impl TestState { Infallible, FeeRuleT::Error, ReceivedNoteId, + PoolType, >, > where diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 6bf84cf007..83fb3e0654 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -465,7 +465,7 @@ pub(crate) mod tests { error::Error, wallet::input_selection::{GreedyInputSelector, GreedyInputSelectorError}, AccountBirthday, Ratio, ShieldedProtocol, WalletCommitmentTrees, WalletRead, - WalletWrite, + WalletWrite, PoolType, }, decrypt_transaction, fees::{fixed, zip317, DustOutputPolicy}, @@ -983,6 +983,7 @@ pub(crate) mod tests { GreedyInputSelectorError, Infallible, ReceivedNoteId, + PoolType, >, > { let txid = st.create_spend_to_address( From c4175342ace3549a4e8ae0bfc6a088c4621e85aa Mon Sep 17 00:00:00 2001 From: Matthew Watt Date: Mon, 9 Oct 2023 06:46:12 -0500 Subject: [PATCH 0233/1122] Cleanup --- zcash_client_backend/src/data_api/error.rs | 23 ++++++++++----------- zcash_client_backend/src/data_api/wallet.rs | 6 +----- zcash_client_sqlite/src/testing.rs | 6 +----- zcash_client_sqlite/src/wallet.rs | 2 +- zcash_client_sqlite/src/wallet/sapling.rs | 3 +-- 5 files changed, 15 insertions(+), 25 deletions(-) diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index 88357816fe..fff7a730ff 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -15,6 +15,7 @@ use zcash_primitives::{ zip32::AccountId, }; +use crate::data_api::PoolType; use crate::data_api::wallet::input_selection::InputSelectorError; #[cfg(feature = "transparent-inputs")] @@ -22,7 +23,7 @@ use zcash_primitives::{legacy::TransparentAddress, zip32::DiversifierIndex}; /// Errors that can occur as a consequence of wallet operations. #[derive(Debug)] -pub enum Error { +pub enum Error { /// An error occurred retrieving data from the underlying data source DataSource(DataSourceError), @@ -68,14 +69,13 @@ pub enum Error fmt::Display for Error +impl fmt::Display for Error where DE: fmt::Display, CE: fmt::Display, SE: fmt::Display, FE: fmt::Display, - N: fmt::Display, - PT: fmt::Display + N: fmt::Display { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match &self { @@ -134,14 +134,13 @@ where } } -impl error::Error for Error +impl error::Error for Error where DE: Debug + Display + error::Error + 'static, CE: Debug + Display + error::Error + 'static, SE: Debug + Display + error::Error + 'static, FE: Debug + Display + 'static, N: Debug + Display, - PT: Debug + Display { fn source(&self) -> Option<&(dyn error::Error + 'static)> { match &self { @@ -154,19 +153,19 @@ where } } -impl From> for Error { +impl From> for Error { fn from(e: builder::Error) -> Self { Error::Builder(e) } } -impl From for Error { +impl From for Error { fn from(e: BalanceError) -> Self { Error::BalanceError(e) } } -impl From> for Error { +impl From> for Error { fn from(e: InputSelectorError) -> Self { match e { InputSelectorError::DataSource(e) => Error::DataSource(e), @@ -183,19 +182,19 @@ impl From> for Error From for Error { +impl From for Error { fn from(e: sapling::builder::Error) -> Self { Error::Builder(builder::Error::SaplingBuild(e)) } } -impl From for Error { +impl From for Error { fn from(e: transparent::builder::Error) -> Self { Error::Builder(builder::Error::TransparentBuild(e)) } } -impl From> for Error { +impl From> for Error { fn from(e: ShardTreeError) -> Self { Error::CommitmentTree(e) } diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index f58fd0be23..ed8c3ec3cd 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -207,7 +207,6 @@ pub fn create_spend_to_address( GreedyInputSelectorError, Infallible, DbT::NoteRef, - PoolType, >, > where @@ -311,7 +310,6 @@ pub fn spend( InputsT::Error, ::Error, DbT::NoteRef, - PoolType, >, > where @@ -366,7 +364,6 @@ pub fn propose_transfer( InputsT::Error, ::Error, DbT::NoteRef, - PoolType, >, > where @@ -450,7 +447,6 @@ pub fn create_proposed_transaction( InputsErrT, FeeRuleT::Error, DbT::NoteRef, - PoolType, >, > where @@ -496,7 +492,7 @@ where let checkpoint_depth = wallet_db.get_checkpoint_depth(min_confirmations)?; - wallet_db.with_sapling_tree_mut::<_, _, Error<_, _, _, _, _, _>>(|sapling_tree| { + wallet_db.with_sapling_tree_mut::<_, _, Error<_, _, _, _, _>>(|sapling_tree| { for selected in proposal.sapling_inputs() { let (note, key, merkle_path) = select_key_for_note( sapling_tree, diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 0a2db28e78..386543c10c 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -15,7 +15,7 @@ use tempfile::NamedTempFile; use tempfile::TempDir; use zcash_client_backend::data_api::chain::ScanSummary; -use zcash_client_backend::data_api::{AccountBalance, WalletRead, PoolType}; +use zcash_client_backend::data_api::{AccountBalance, WalletRead}; #[allow(deprecated)] use zcash_client_backend::{ address::RecipientAddress, @@ -446,7 +446,6 @@ impl TestState { GreedyInputSelectorError, Infallible, ReceivedNoteId, - PoolType, >, > { let params = self.network(); @@ -480,7 +479,6 @@ impl TestState { InputsT::Error, ::Error, ReceivedNoteId, - PoolType, >, > where @@ -515,7 +513,6 @@ impl TestState { InputsT::Error, ::Error, ReceivedNoteId, - PoolType, >, > where @@ -582,7 +579,6 @@ impl TestState { Infallible, FeeRuleT::Error, ReceivedNoteId, - PoolType, >, > where diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 41ea48e0f0..524d3431a1 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -136,7 +136,7 @@ pub(crate) fn pool_code(pool_type: PoolType) -> i64 { match pool_type { PoolType::Transparent => 0i64, PoolType::Shielded(ShieldedProtocol::Sapling) => 2i64, - PoolType::Shielded(ShieldedProtocol::Orchard) => 4i64 + PoolType::Shielded(ShieldedProtocol::Orchard) => 3i64 } } diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 83fb3e0654..6bf84cf007 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -465,7 +465,7 @@ pub(crate) mod tests { error::Error, wallet::input_selection::{GreedyInputSelector, GreedyInputSelectorError}, AccountBirthday, Ratio, ShieldedProtocol, WalletCommitmentTrees, WalletRead, - WalletWrite, PoolType, + WalletWrite, }, decrypt_transaction, fees::{fixed, zip317, DustOutputPolicy}, @@ -983,7 +983,6 @@ pub(crate) mod tests { GreedyInputSelectorError, Infallible, ReceivedNoteId, - PoolType, >, > { let txid = st.create_spend_to_address( From a910fb86b6e2cd24e394731fe33d448c5eabb0d9 Mon Sep 17 00:00:00 2001 From: Matthew Watt Date: Mon, 9 Oct 2023 06:57:40 -0500 Subject: [PATCH 0234/1122] Revert unnecessary change --- zcash_client_backend/src/data_api/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index fff7a730ff..be60adc200 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -75,7 +75,7 @@ where CE: fmt::Display, SE: fmt::Display, FE: fmt::Display, - N: fmt::Display + N: fmt::Display, { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match &self { From 4bbe658f74fc31b276bc41d8b3e9f6b12f3f8ee7 Mon Sep 17 00:00:00 2001 From: Matthew Watt <66689851+tw0po1nt@users.noreply.github.com> Date: Mon, 9 Oct 2023 10:06:37 -0500 Subject: [PATCH 0235/1122] Update CHANGELOG.md Co-authored-by: Daira Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 0af57a371b..e95e32e36b 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -8,7 +8,9 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added -- `zcash_client_backend::data_api::ShieldedProtocol` has a new variant for `Orchard`, allowing for better reporting to callers trying to perform actions using `Orchard` before it is fully supported. +- `zcash_client_backend::data_api::ShieldedProtocol` has a new variant for `Orchard`, + allowing for better reporting to callers trying to perform actions using `Orchard` + before it is fully supported. - `zcash_client_backend::data_api::error::Error` has new error variant: - `Error::UnsupportedPoolType(zcash_client_backend::data_api::PoolType)` From 13a2d5d7d469225d69c7af6453effe4cd2453fbd Mon Sep 17 00:00:00 2001 From: Matthew Watt <66689851+tw0po1nt@users.noreply.github.com> Date: Mon, 9 Oct 2023 10:07:31 -0500 Subject: [PATCH 0236/1122] Update wallet.rs Co-authored-by: Daira Emma Hopwood --- zcash_client_backend/src/data_api/wallet.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index ed8c3ec3cd..587d616ad0 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -426,8 +426,8 @@ where /// Returns the database identifier for the newly constructed transaction, or an error if /// an error occurs in transaction construction, proving, or signing. /// -/// Note: If the payment includes a recipient with an Orchard-only UA, this will attempt to fallback -/// to the transparent receiver until full Orchard support is implemented +/// Note: If the payment includes a recipient with an Orchard-only UA, this will attempt +/// to fall back to the transparent receiver until full Orchard support is implemented. #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] pub fn create_proposed_transaction( From 74840829c807ce3c3d0df978d76026423bac44dd Mon Sep 17 00:00:00 2001 From: sasha Date: Mon, 22 May 2023 16:08:20 -0700 Subject: [PATCH 0237/1122] zcash_client_backend: Make `ReceivedSaplingNote` internals private. This also adds the txid and index to `ReceivedSaplingNote` for use as a globally identifier for the note. --- zcash_client_backend/CHANGELOG.md | 8 +++ zcash_client_backend/src/data_api/wallet.rs | 15 +++-- .../src/data_api/wallet/input_selection.rs | 2 +- zcash_client_backend/src/wallet.rs | 57 +++++++++++++++++-- zcash_client_sqlite/src/wallet/sapling.rs | 26 +++++---- zcash_primitives/src/transaction/mod.rs | 6 ++ 6 files changed, 89 insertions(+), 25 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 449efeb5f3..9bf7fe75e5 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -7,9 +7,17 @@ and this library adheres to Rust's notion of ## [Unreleased] +### Added + +- Added methods to `zcash_client_backend::wallet::ReceivedSaplingNote`: + `{from_parts, txid, output_index, diversifier, rseed, note_commitment_tree_position}`. + ### Changed - `zcash_client_backend::data_api::chain::scan_cached_blocks` now returns a `ScanSummary` containing metadata about the scanned blocks on success. +- The fields of `zcash_client_backend::wallet::ReceivedSaplingNote` are now + private. Use `ReceivedSaplingNote::from_parts` for construction instead. + Accessor methods are provided for each previously-public field. ## [0.10.0] - 2023-09-25 diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 0d1adda6fe..f4373d8bb6 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -498,9 +498,8 @@ where &dfvk, checkpoint_depth, )? - .ok_or(Error::NoteMismatch(selected.note_id))?; - - builder.add_sapling_spend(key, selected.diversifier, note, merkle_path)?; + .ok_or(Error::NoteMismatch(selected.internal_note_id().clone()))?; + builder.add_sapling_spend(key, selected.diversifier(), note, merkle_path)?; } Ok(()) })?; @@ -774,15 +773,15 @@ fn select_key_for_note>( // corresponding to the unified spending key, checking against the witness we are using // to spend the note that we've used the correct key. let external_note = dfvk - .diversified_address(selected.diversifier) - .map(|addr| addr.create_note(selected.note_value.into(), selected.rseed)); + .diversified_address(selected.diversifier()) + .map(|addr| addr.create_note(selected.value().into(), selected.rseed())); let internal_note = dfvk - .diversified_change_address(selected.diversifier) - .map(|addr| addr.create_note(selected.note_value.into(), selected.rseed)); + .diversified_change_address(selected.diversifier()) + .map(|addr| addr.create_note(selected.value().into(), selected.rseed())); let expected_root = commitment_tree.root_at_checkpoint(checkpoint_depth)?; let merkle_path = commitment_tree - .witness_caching(selected.note_commitment_tree_position, checkpoint_depth)?; + .witness_caching(selected.note_commitment_tree_position(), checkpoint_depth)?; Ok(external_note .filter(|n| expected_root == merkle_path.root(Node::from_cmu(&n.cmu()))) diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 578082a337..0c575d9a09 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -419,7 +419,7 @@ where let new_available = sapling_inputs .iter() - .map(|n| n.note_value) + .map(|n| n.value()) .sum::>() .ok_or(BalanceError::Overflow)?; diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index ccfe5a75b0..c7467c8a69 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -177,11 +177,58 @@ impl WalletSaplingOutput { /// with sufficient information for use in note selection. #[derive(Debug)] pub struct ReceivedSaplingNote { - pub note_id: NoteRef, - pub diversifier: sapling::Diversifier, - pub note_value: Amount, - pub rseed: sapling::Rseed, - pub note_commitment_tree_position: Position, + note_id: NoteRef, + txid: TxId, + output_index: u16, + diversifier: sapling::Diversifier, + note_value: Amount, + rseed: sapling::Rseed, + note_commitment_tree_position: Position, +} + +impl ReceivedSaplingNote { + pub fn from_parts( + note_id: NoteRef, + txid: TxId, + output_index: u16, + diversifier: sapling::Diversifier, + note_value: Amount, + rseed: sapling::Rseed, + note_commitment_tree_position: Position, + ) -> Self { + ReceivedSaplingNote { + note_id, + txid, + output_index, + diversifier, + note_value, + rseed, + note_commitment_tree_position, + } + } + + pub fn internal_note_id(&self) -> &NoteRef { + &self.note_id + } + + pub fn txid(&self) -> &TxId { + &self.txid + } + pub fn output_index(&self) -> u16 { + self.output_index + } + pub fn diversifier(&self) -> sapling::Diversifier { + self.diversifier + } + pub fn value(&self) -> Amount { + self.note_value + } + pub fn rseed(&self) -> sapling::Rseed { + self.rseed + } + pub fn note_commitment_tree_position(&self) -> Position { + self.note_commitment_tree_position + } } impl sapling_fees::InputView for ReceivedSaplingNote { diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 6bf84cf007..1fa831f415 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -9,7 +9,7 @@ use zcash_primitives::{ consensus::BlockHeight, memo::MemoBytes, sapling::{self, Diversifier, Note, Nullifier, Rseed}, - transaction::components::Amount, + transaction::{components::Amount, TxId}, zip32::AccountId, }; @@ -83,8 +83,10 @@ impl ReceivedSaplingOutput for DecryptedOutput { fn to_spendable_note(row: &Row) -> Result, SqliteClientError> { let note_id = ReceivedNoteId(row.get(0)?); + let txid = row.get::<_, [u8; 32]>(1).map(TxId::from_bytes)?; + let output_index = row.get(2)?; let diversifier = { - let d: Vec<_> = row.get(1)?; + let d: Vec<_> = row.get(3)?; if d.len() != 11 { return Err(SqliteClientError::CorruptedData( "Invalid diversifier length".to_string(), @@ -95,10 +97,10 @@ fn to_spendable_note(row: &Row) -> Result, S Diversifier(tmp) }; - let note_value = Amount::from_i64(row.get(2)?).unwrap(); + let note_value = Amount::from_i64(row.get(4)?).unwrap(); let rseed = { - let rcm_bytes: Vec<_> = row.get(3)?; + let rcm_bytes: Vec<_> = row.get(5)?; // We store rcm directly in the data DB, regardless of whether the note // used a v1 or v2 note plaintext, so for the purposes of spending let's @@ -113,17 +115,19 @@ fn to_spendable_note(row: &Row) -> Result, S }; let note_commitment_tree_position = - Position::from(u64::try_from(row.get::<_, i64>(4)?).map_err(|_| { + Position::from(u64::try_from(row.get::<_, i64>(6)?).map_err(|_| { SqliteClientError::CorruptedData("Note commitment tree position invalid.".to_string()) })?); - Ok(ReceivedSaplingNote { + Ok(ReceivedSaplingNote::from_parts( note_id, + txid, + output_index, diversifier, note_value, rseed, note_commitment_tree_position, - }) + )) } /// Utility method for determining whether we have any spendable notes @@ -170,7 +174,7 @@ pub(crate) fn get_spendable_sapling_notes( } let mut stmt_select_notes = conn.prepare_cached( - "SELECT id_note, diversifier, value, rcm, commitment_tree_position + "SELECT id_note, txid, output_index, diversifier, value, rcm, commitment_tree_position FROM sapling_received_notes INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx WHERE account = :account @@ -244,7 +248,7 @@ pub(crate) fn select_spendable_sapling_notes( // 4) Match the selected notes against the witnesses at the desired height. let mut stmt_select_notes = conn.prepare_cached( "WITH eligible AS ( - SELECT id_note, diversifier, value, rcm, commitment_tree_position, + SELECT id_note, txid, output_index, diversifier, value, rcm, commitment_tree_position, SUM(value) OVER (PARTITION BY account, spent ORDER BY id_note) AS so_far FROM sapling_received_notes @@ -266,10 +270,10 @@ pub(crate) fn select_spendable_sapling_notes( AND unscanned.block_range_end > :wallet_birthday ) ) - SELECT id_note, diversifier, value, rcm, commitment_tree_position + SELECT id_note, txid, output_index, diversifier, value, rcm, commitment_tree_position FROM eligible WHERE so_far < :target_value UNION - SELECT id_note, diversifier, value, rcm, commitment_tree_position + SELECT id_note, txid, output_index, diversifier, value, rcm, commitment_tree_position FROM (SELECT * from eligible WHERE so_far >= :target_value LIMIT 1)", )?; diff --git a/zcash_primitives/src/transaction/mod.rs b/zcash_primitives/src/transaction/mod.rs index 5d3ebf365e..f44a3849d2 100644 --- a/zcash_primitives/src/transaction/mod.rs +++ b/zcash_primitives/src/transaction/mod.rs @@ -92,6 +92,12 @@ impl AsRef<[u8; 32]> for TxId { } } +impl From for [u8; 32] { + fn from(value: TxId) -> Self { + value.0 + } +} + impl TxId { pub fn from_bytes(bytes: [u8; 32]) -> Self { TxId(bytes) From 5b40ddf0723509dbd5d67d9bbcc6b4198a582e9c Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 9 Oct 2023 11:44:41 -0600 Subject: [PATCH 0238/1122] zcash_client_backend: Use globally unique identifiers for notes. Update zcash_client_backend error types to use (TxId, output_index) as the identifier for notes instead of the internal database identifier. --- zcash_client_backend/CHANGELOG.md | 4 +++ zcash_client_backend/src/data_api/error.rs | 26 ++++++++-------- zcash_client_backend/src/data_api/wallet.rs | 34 ++++++++------------- zcash_client_sqlite/src/testing.rs | 6 ---- zcash_client_sqlite/src/wallet/sapling.rs | 1 - 5 files changed, 29 insertions(+), 42 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 9bf7fe75e5..da8ea4f01d 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -18,6 +18,10 @@ and this library adheres to Rust's notion of - The fields of `zcash_client_backend::wallet::ReceivedSaplingNote` are now private. Use `ReceivedSaplingNote::from_parts` for construction instead. Accessor methods are provided for each previously-public field. +- `zcash_client_backend::data_api` changes: + - The `NoteMismatch` variant of `data_api::error::Error` now wraps a + `data_api::NoteId` instead of a backend-specific note identifier. The + related `NoteRef` type parameter has been removed from `data_api::error::Error`. ## [0.10.0] - 2023-09-25 diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index 31a9b39c99..a982cece50 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -20,9 +20,11 @@ use crate::data_api::wallet::input_selection::InputSelectorError; #[cfg(feature = "transparent-inputs")] use zcash_primitives::{legacy::TransparentAddress, zip32::DiversifierIndex}; +use super::NoteId; + /// Errors that can occur as a consequence of wallet operations. #[derive(Debug)] -pub enum Error { +pub enum Error { /// An error occurred retrieving data from the underlying data source DataSource(DataSourceError), @@ -56,7 +58,7 @@ pub enum Error fmt::Display for Error +impl fmt::Display for Error where DE: fmt::Display, CE: fmt::Display, SE: fmt::Display, FE: fmt::Display, - N: fmt::Display, { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match &self { @@ -111,7 +112,7 @@ where Error::ScanRequired => write!(f, "Must scan blocks first"), Error::Builder(e) => write!(f, "An error occurred building the transaction: {}", e), Error::MemoForbidden => write!(f, "It is not possible to send a memo to a transparent address."), - Error::NoteMismatch(n) => write!(f, "A note being spent ({}) does not correspond to either the internal or external full viewing key for the provided spending key.", n), + Error::NoteMismatch(n) => write!(f, "A note being spent ({:?}) does not correspond to either the internal or external full viewing key for the provided spending key.", n), #[cfg(feature = "transparent-inputs")] Error::AddressNotRecognized(_) => { @@ -129,13 +130,12 @@ where } } -impl error::Error for Error +impl error::Error for Error where DE: Debug + Display + error::Error + 'static, CE: Debug + Display + error::Error + 'static, SE: Debug + Display + error::Error + 'static, FE: Debug + Display + 'static, - N: Debug + Display, { fn source(&self) -> Option<&(dyn error::Error + 'static)> { match &self { @@ -148,19 +148,19 @@ where } } -impl From> for Error { +impl From> for Error { fn from(e: builder::Error) -> Self { Error::Builder(e) } } -impl From for Error { +impl From for Error { fn from(e: BalanceError) -> Self { Error::BalanceError(e) } } -impl From> for Error { +impl From> for Error { fn from(e: InputSelectorError) -> Self { match e { InputSelectorError::DataSource(e) => Error::DataSource(e), @@ -177,19 +177,19 @@ impl From> for Error From for Error { +impl From for Error { fn from(e: sapling::builder::Error) -> Self { Error::Builder(builder::Error::SaplingBuild(e)) } } -impl From for Error { +impl From for Error { fn from(e: transparent::builder::Error) -> Self { Error::Builder(builder::Error::TransparentBuild(e)) } } -impl From> for Error { +impl From> for Error { fn from(e: ShardTreeError) -> Self { Error::CommitmentTree(e) } diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index f4373d8bb6..7da3dc8735 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -37,7 +37,7 @@ use crate::{ pub mod input_selection; use input_selection::{GreedyInputSelector, GreedyInputSelectorError, InputSelector}; -use super::ShieldedProtocol; +use super::{NoteId, ShieldedProtocol}; #[cfg(feature = "transparent-inputs")] use { @@ -206,7 +206,6 @@ pub fn create_spend_to_address( ::Error, GreedyInputSelectorError, Infallible, - DbT::NoteRef, >, > where @@ -309,7 +308,6 @@ pub fn spend( ::Error, InputsT::Error, ::Error, - DbT::NoteRef, >, > where @@ -358,13 +356,7 @@ pub fn propose_transfer( min_confirmations: NonZeroU32, ) -> Result< Proposal, - Error< - DbT::Error, - CommitmentTreeErrT, - InputsT::Error, - ::Error, - DbT::NoteRef, - >, + Error::Error>, > where DbT: WalletWrite, @@ -395,13 +387,7 @@ pub fn propose_shielding( min_confirmations: NonZeroU32, ) -> Result< Proposal, - Error< - DbT::Error, - CommitmentTreeErrT, - InputsT::Error, - ::Error, - DbT::NoteRef, - >, + Error::Error>, > where ParamsT: consensus::Parameters, @@ -443,7 +429,6 @@ pub fn create_proposed_transaction( ::Error, InputsErrT, FeeRuleT::Error, - DbT::NoteRef, >, > where @@ -488,8 +473,7 @@ where let mut builder = Builder::new(params.clone(), proposal.min_target_height(), None); let checkpoint_depth = wallet_db.get_checkpoint_depth(min_confirmations)?; - - wallet_db.with_sapling_tree_mut::<_, _, Error<_, _, _, _, _>>(|sapling_tree| { + wallet_db.with_sapling_tree_mut::<_, _, Error<_, _, _, _>>(|sapling_tree| { for selected in proposal.sapling_inputs() { let (note, key, merkle_path) = select_key_for_note( sapling_tree, @@ -498,7 +482,14 @@ where &dfvk, checkpoint_depth, )? - .ok_or(Error::NoteMismatch(selected.internal_note_id().clone()))?; + .ok_or_else(|| { + Error::NoteMismatch(NoteId { + txid: *selected.txid(), + protocol: ShieldedProtocol::Sapling, + output_index: selected.output_index(), + }) + })?; + builder.add_sapling_spend(key, selected.diversifier(), note, merkle_path)?; } Ok(()) @@ -724,7 +715,6 @@ pub fn shield_transparent_funds( ::Error, InputsT::Error, ::Error, - DbT::NoteRef, >, > where diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index 386543c10c..f6a9bbc7eb 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -445,7 +445,6 @@ impl TestState { commitment_tree::Error, GreedyInputSelectorError, Infallible, - ReceivedNoteId, >, > { let params = self.network(); @@ -478,7 +477,6 @@ impl TestState { commitment_tree::Error, InputsT::Error, ::Error, - ReceivedNoteId, >, > where @@ -512,7 +510,6 @@ impl TestState { Infallible, InputsT::Error, ::Error, - ReceivedNoteId, >, > where @@ -546,7 +543,6 @@ impl TestState { Infallible, InputsT::Error, ::Error, - ReceivedNoteId, >, > where @@ -578,7 +574,6 @@ impl TestState { commitment_tree::Error, Infallible, FeeRuleT::Error, - ReceivedNoteId, >, > where @@ -615,7 +610,6 @@ impl TestState { commitment_tree::Error, InputsT::Error, ::Error, - ReceivedNoteId, >, > where diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 1fa831f415..db5b6bb851 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -986,7 +986,6 @@ pub(crate) mod tests { commitment_tree::Error, GreedyInputSelectorError, Infallible, - ReceivedNoteId, >, > { let txid = st.create_spend_to_address( From 4bd3259ad05e7313a475cae6dee6b8f468bdc372 Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 5 Oct 2023 20:14:10 -0600 Subject: [PATCH 0239/1122] Add a MARGINAL_FEE constant for the ZIP 317 marginal fee --- zcash_primitives/CHANGELOG.md | 2 ++ zcash_primitives/src/transaction/fees/zip317.rs | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 20bb1e4d26..a1fb27e0ba 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -21,6 +21,8 @@ and this library adheres to Rust's notion of set of closures. - Test helpers, behind the `test-dependencies` feature flag: - `zcash_primitives::prover::mock::{MockSpendProver, MockOutputProver}` +- MARGINAL_FEE pub const Amount: + - `zcash_primitives::transaction::fees::zip317::MARGINAL_FEE` ### Changed - `zcash_primitives::transaction::components::sapling`: diff --git a/zcash_primitives/src/transaction/fees/zip317.rs b/zcash_primitives/src/transaction/fees/zip317.rs index 21ecf4cb95..d1bb5efd8b 100644 --- a/zcash_primitives/src/transaction/fees/zip317.rs +++ b/zcash_primitives/src/transaction/fees/zip317.rs @@ -20,6 +20,11 @@ use crate::{ /// [ZIP 317]: https//zips.z.cash/zip-0317 pub const MINIMUM_FEE: Amount = Amount::const_from_i64(10_000); +/// The marginal fee using the standard [ZIP 317] constants. +/// +/// [ZIP 317]: https//zips.z.cash/zip-0317 +pub const MARGINAL_FEE: Amount = Amount::const_from_i64(5_000); + /// A [`FeeRule`] implementation that implements the [ZIP 317] fee rule. /// /// This fee rule supports only P2pkh transparent inputs; an error will be returned if a coin @@ -42,7 +47,7 @@ impl FeeRule { /// [ZIP 317]: https//zips.z.cash/zip-0317 pub fn standard() -> Self { Self { - marginal_fee: Amount::from_u64(5000).unwrap(), + marginal_fee: MARGINAL_FEE, grace_actions: 2, p2pkh_standard_input_size: 150, p2pkh_standard_output_size: 34, From 5de3fbb36ba761e500f3160abeb3a48b8e48ccef Mon Sep 17 00:00:00 2001 From: Matthew Watt Date: Tue, 10 Oct 2023 06:35:04 -0500 Subject: [PATCH 0240/1122] Fix logic error --- zcash_client_backend/src/data_api/wallet.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 587d616ad0..13cd767839 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -557,6 +557,11 @@ where payment.amount, memo.clone(), )?; + sapling_output_meta.push(( + Recipient::Unified(ua.clone(), PoolType::Shielded(ShieldedProtocol::Sapling)), + payment.amount, + Some(memo), + )); } else if let Some(taddr) = ua.transparent() { if payment.memo.is_some() { return Err(Error::MemoForbidden); @@ -569,11 +574,6 @@ where } else { return Err(Error::UnsupportedPoolType(PoolType::Shielded(ShieldedProtocol::Orchard))); } - sapling_output_meta.push(( - Recipient::Unified(ua.clone(), PoolType::Shielded(ShieldedProtocol::Sapling)), - payment.amount, - Some(memo), - )); } RecipientAddress::Shielded(addr) => { let memo = payment From 0e5a58b0b04689ef5473976b7370dec523588349 Mon Sep 17 00:00:00 2001 From: Matthew Watt <66689851+tw0po1nt@users.noreply.github.com> Date: Tue, 10 Oct 2023 06:36:20 -0500 Subject: [PATCH 0241/1122] Update zcash_client_sqlite/src/wallet.rs Co-authored-by: Daira Emma Hopwood --- zcash_client_sqlite/src/wallet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 524d3431a1..bee78f3d62 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -779,7 +779,7 @@ pub(crate) fn get_received_memo( ) .optional()? .flatten(), - ShieldedProtocol::Orchard => None + _ => return Err(SqliteClientError::UnsupportedPoolType(PoolType::Shielded(protocol))); }; memo_bytes From b548e00c107c00ba7c8d11b9566ad33faee65d24 Mon Sep 17 00:00:00 2001 From: Matthew Watt <66689851+tw0po1nt@users.noreply.github.com> Date: Tue, 10 Oct 2023 06:36:40 -0500 Subject: [PATCH 0242/1122] Update zcash_client_sqlite/src/wallet.rs Co-authored-by: Daira Emma Hopwood --- zcash_client_sqlite/src/wallet.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index bee78f3d62..896d9fa6d9 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -760,10 +760,6 @@ pub(crate) fn get_received_memo( note_id: NoteId, ) -> Result, SqliteClientError> { let protocol = note_id.protocol(); - if let ShieldedProtocol::Orchard = protocol { - return Err(SqliteClientError::UnsupportedPoolType(PoolType::Shielded(protocol))); - } - let memo_bytes: Option> = match protocol { ShieldedProtocol::Sapling => conn .query_row( From 2a4d9e06da233825aeb22e7617bde8f60df01dfa Mon Sep 17 00:00:00 2001 From: Matthew Watt Date: Tue, 10 Oct 2023 07:47:08 -0500 Subject: [PATCH 0243/1122] Run fmt --- zcash_client_backend/src/data_api.rs | 2 +- zcash_client_backend/src/data_api/error.rs | 2 +- zcash_client_backend/src/data_api/wallet.rs | 16 +++++++++------- zcash_client_sqlite/src/error.rs | 2 +- zcash_client_sqlite/src/wallet.rs | 8 ++++++-- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 7be3321c5a..0291700d5c 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -557,7 +557,7 @@ pub enum ShieldedProtocol { /// The Sapling protocol Sapling, /// The Orchard protocol - Orchard + Orchard, } /// A unique identifier for a shielded transaction output diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index be60adc200..c6105ad87e 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -15,8 +15,8 @@ use zcash_primitives::{ zip32::AccountId, }; -use crate::data_api::PoolType; use crate::data_api::wallet::input_selection::InputSelectorError; +use crate::data_api::PoolType; #[cfg(feature = "transparent-inputs")] use zcash_primitives::{legacy::TransparentAddress, zip32::DiversifierIndex}; diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 13cd767839..6b64ab142c 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -425,7 +425,7 @@ where /// /// Returns the database identifier for the newly constructed transaction, or an error if /// an error occurs in transaction construction, proving, or signing. -/// +/// /// Note: If the payment includes a recipient with an Orchard-only UA, this will attempt /// to fall back to the transparent receiver until full Orchard support is implemented. #[allow(clippy::too_many_arguments)] @@ -558,7 +558,10 @@ where memo.clone(), )?; sapling_output_meta.push(( - Recipient::Unified(ua.clone(), PoolType::Shielded(ShieldedProtocol::Sapling)), + Recipient::Unified( + ua.clone(), + PoolType::Shielded(ShieldedProtocol::Sapling), + ), payment.amount, Some(memo), )); @@ -566,13 +569,12 @@ where if payment.memo.is_some() { return Err(Error::MemoForbidden); } else { - builder.add_transparent_output( - taddr, - payment.amount - )?; + builder.add_transparent_output(taddr, payment.amount)?; } } else { - return Err(Error::UnsupportedPoolType(PoolType::Shielded(ShieldedProtocol::Orchard))); + return Err(Error::UnsupportedPoolType(PoolType::Shielded( + ShieldedProtocol::Orchard, + ))); } } RecipientAddress::Shielded(addr) => { diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 4d9977d762..82104d2855 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -101,7 +101,7 @@ pub enum SqliteClientError { ChainHeightUnknown, /// Unsupported pool type - UnsupportedPoolType(PoolType) + UnsupportedPoolType(PoolType), } impl error::Error for SqliteClientError { diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 896d9fa6d9..35d9c0a847 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -136,7 +136,7 @@ pub(crate) fn pool_code(pool_type: PoolType) -> i64 { match pool_type { PoolType::Transparent => 0i64, PoolType::Shielded(ShieldedProtocol::Sapling) => 2i64, - PoolType::Shielded(ShieldedProtocol::Orchard) => 3i64 + PoolType::Shielded(ShieldedProtocol::Orchard) => 3i64, } } @@ -775,7 +775,11 @@ pub(crate) fn get_received_memo( ) .optional()? .flatten(), - _ => return Err(SqliteClientError::UnsupportedPoolType(PoolType::Shielded(protocol))); + _ => { + return Err(SqliteClientError::UnsupportedPoolType(PoolType::Shielded( + protocol, + ))) + } }; memo_bytes From 73f263682ec2715da3804bb134aab68cdc5f63ca Mon Sep 17 00:00:00 2001 From: Matthew Watt Date: Tue, 10 Oct 2023 07:50:07 -0500 Subject: [PATCH 0244/1122] Remove unnecessary local --- zcash_client_sqlite/src/wallet.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 35d9c0a847..01ec36faa2 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -759,8 +759,7 @@ pub(crate) fn get_received_memo( conn: &rusqlite::Connection, note_id: NoteId, ) -> Result, SqliteClientError> { - let protocol = note_id.protocol(); - let memo_bytes: Option> = match protocol { + let memo_bytes: Option> = match note_id.protocol() { ShieldedProtocol::Sapling => conn .query_row( "SELECT memo FROM sapling_received_notes @@ -777,7 +776,7 @@ pub(crate) fn get_received_memo( .flatten(), _ => { return Err(SqliteClientError::UnsupportedPoolType(PoolType::Shielded( - protocol, + note_id.protocol(), ))) } }; From a2b5c2c7841e7249ec56f73faa88ae6bba028cd1 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 28 Aug 2023 19:09:14 -0600 Subject: [PATCH 0245/1122] zcash_client_backend: Use `NonNegativeAmount` for fee and change amounts. In order to use `uint64` for amounts that must be nonnegative in the `proposal.proto` file, it is useful to update fee and change computation to use `NonNegativeAmount` where possible. --- zcash_client_backend/CHANGELOG.md | 6 + zcash_client_backend/src/data_api.rs | 1 + zcash_client_backend/src/data_api/wallet.rs | 21 ++-- .../src/data_api/wallet/input_selection.rs | 4 +- zcash_client_backend/src/fees.rs | 45 ++++---- zcash_client_backend/src/fees/fixed.rs | 40 ++++--- zcash_client_backend/src/fees/zip317.rs | 39 ++++--- zcash_client_backend/src/scanning.rs | 12 +- zcash_client_sqlite/src/chain.rs | 54 +++++---- zcash_client_sqlite/src/lib.rs | 8 +- zcash_client_sqlite/src/testing.rs | 27 ++--- zcash_client_sqlite/src/wallet.rs | 6 +- zcash_client_sqlite/src/wallet/sapling.rs | 107 +++++++++--------- zcash_client_sqlite/src/wallet/scanning.rs | 10 +- zcash_extensions/src/transparent/demo.rs | 4 +- zcash_primitives/CHANGELOG.md | 37 +++++- .../src/sapling/note_encryption.rs | 21 +++- zcash_primitives/src/transaction/builder.rs | 66 ++++++----- .../src/transaction/components/amount.rs | 67 ++++++++++- zcash_primitives/src/transaction/fees.rs | 6 +- .../src/transaction/fees/fixed.rs | 12 +- .../src/transaction/fees/zip317.rs | 42 ++++--- 22 files changed, 383 insertions(+), 252 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index da8ea4f01d..d734954ec2 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -22,6 +22,12 @@ and this library adheres to Rust's notion of - The `NoteMismatch` variant of `data_api::error::Error` now wraps a `data_api::NoteId` instead of a backend-specific note identifier. The related `NoteRef` type parameter has been removed from `data_api::error::Error`. + - `wallet::create_spend_to_address` now takes a `NonNegativeAmount` rather than + an `Amount`. + - All uses of `Amount` in `data_api::wallet::input_selection` have been replaced + with `NonNegativeAmount`. +- All uses of `Amount` in `zcash_client_backend::fees` have been replaced + with `NonNegativeAmount`. ## [0.10.0] - 2023-09-25 diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 875ac614b3..ccf677d93b 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -640,6 +640,7 @@ impl SentTransactionOutput { sapling_change_to, } } + /// Returns the index within the transaction that contains the recipient output. /// /// - If `recipient_address` is a Sapling address, this is an index into the Sapling diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 7da3dc8735..ddc5b5c38d 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -13,7 +13,7 @@ use zcash_primitives::{ }, transaction::{ builder::Builder, - components::amount::{Amount, BalanceError}, + components::amount::{Amount, BalanceError, NonNegativeAmount}, fees::{fixed, FeeRule}, Transaction, }, @@ -42,10 +42,7 @@ use super::{NoteId, ShieldedProtocol}; #[cfg(feature = "transparent-inputs")] use { crate::wallet::WalletTransparentOutput, - zcash_primitives::{ - legacy::TransparentAddress, sapling::keys::OutgoingViewingKey, - transaction::components::amount::NonNegativeAmount, - }, + zcash_primitives::{legacy::TransparentAddress, sapling::keys::OutgoingViewingKey}, }; /// Scans a [`Transaction`] for any information that can be decrypted by the accounts in @@ -195,7 +192,7 @@ pub fn create_spend_to_address( prover: impl SaplingProver, usk: &UnifiedSpendingKey, to: &RecipientAddress, - amount: Amount, + amount: NonNegativeAmount, memo: Option, ovk_policy: OvkPolicy, min_confirmations: NonZeroU32, @@ -215,7 +212,7 @@ where { let req = zip321::TransactionRequest::new(vec![Payment { recipient_address: to.clone(), - amount, + amount: amount.into(), memo, label: None, message: None, @@ -576,7 +573,7 @@ where builder.add_sapling_output( internal_ovk(), dfvk.change_address().1, - *amount, + amount.into(), memo.clone(), )?; sapling_output_meta.push(( @@ -584,7 +581,7 @@ where account, PoolType::Shielded(ShieldedProtocol::Sapling), ), - *amount, + amount.into(), Some(memo), )) } @@ -653,7 +650,7 @@ where created: time::OffsetDateTime::now_utc(), account, outputs: sapling_outputs.chain(transparent_outputs).collect(), - fee_amount: proposal.balance().fee_required(), + fee_amount: Amount::from(proposal.balance().fee_required()), #[cfg(feature = "transparent-inputs")] utxos_spent: utxos.iter().map(|utxo| utxo.outpoint().clone()).collect(), }) @@ -764,10 +761,10 @@ fn select_key_for_note>( // to spend the note that we've used the correct key. let external_note = dfvk .diversified_address(selected.diversifier()) - .map(|addr| addr.create_note(selected.value().into(), selected.rseed())); + .map(|addr| addr.create_note(selected.value().try_into().unwrap(), selected.rseed())); let internal_note = dfvk .diversified_change_address(selected.diversifier()) - .map(|addr| addr.create_note(selected.value().into(), selected.rseed())); + .map(|addr| addr.create_note(selected.value().try_into().unwrap(), selected.rseed())); let expected_root = commitment_tree.root_at_checkpoint(checkpoint_depth)?; let merkle_path = commitment_tree diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 0c575d9a09..4bdbede17b 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -493,7 +493,7 @@ where } }; - if balance.total() >= shielding_threshold.into() { + if balance.total() >= shielding_threshold { Ok(Proposal { transaction_request: TransactionRequest::empty(), transparent_inputs, @@ -506,7 +506,7 @@ where }) } else { Err(InputSelectorError::InsufficientFunds { - available: balance.total(), + available: balance.total().into(), required: shielding_threshold.into(), }) } diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index 050f3dbfa1..1681c225b2 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -4,7 +4,7 @@ use zcash_primitives::{ consensus::{self, BlockHeight}, transaction::{ components::{ - amount::{Amount, BalanceError}, + amount::{Amount, BalanceError, NonNegativeAmount}, sapling::fees as sapling, transparent::fees as transparent, OutPoint, @@ -19,11 +19,11 @@ pub mod zip317; /// A proposed change amount and output pool. #[derive(Clone, Debug, PartialEq, Eq)] pub enum ChangeValue { - Sapling(Amount), + Sapling(NonNegativeAmount), } impl ChangeValue { - pub fn value(&self) -> Amount { + pub fn value(&self) -> NonNegativeAmount { match self { ChangeValue::Sapling(value) => *value, } @@ -36,23 +36,28 @@ impl ChangeValue { #[derive(Clone, Debug, PartialEq, Eq)] pub struct TransactionBalance { proposed_change: Vec, - fee_required: Amount, - total: Amount, + fee_required: NonNegativeAmount, + total: NonNegativeAmount, } impl TransactionBalance { /// Constructs a new balance from its constituent parts. - pub fn new(proposed_change: Vec, fee_required: Amount) -> Option { - proposed_change + pub fn new( + proposed_change: Vec, + fee_required: NonNegativeAmount, + ) -> Result { + let total = proposed_change .iter() - .map(|v| v.value()) - .chain(Some(fee_required)) - .sum::>() - .map(|total| TransactionBalance { - proposed_change, - fee_required, - total, - }) + .map(|c| c.value()) + .chain(Some(fee_required).into_iter()) + .sum::>() + .ok_or(())?; + + Ok(Self { + proposed_change, + fee_required, + total, + }) } /// The change values proposed by the [`ChangeStrategy`] that computed this balance. @@ -62,12 +67,12 @@ impl TransactionBalance { /// Returns the fee computed for the transaction, assuming that the suggested /// change outputs are added to the transaction. - pub fn fee_required(&self) -> Amount { + pub fn fee_required(&self) -> NonNegativeAmount { self.fee_required } /// Returns the sum of the proposed change outputs and the required fee. - pub fn total(&self) -> Amount { + pub fn total(&self) -> NonNegativeAmount { self.total } } @@ -147,7 +152,7 @@ pub enum DustAction { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct DustOutputPolicy { action: DustAction, - dust_threshold: Option, + dust_threshold: Option, } impl DustOutputPolicy { @@ -157,7 +162,7 @@ impl DustOutputPolicy { /// of the dust threshold to the change strategy that is evaluating the strategy; this /// recommended, but an explicit value (including zero) may be provided to explicitly /// override the determination of the change strategy. - pub fn new(action: DustAction, dust_threshold: Option) -> Self { + pub fn new(action: DustAction, dust_threshold: Option) -> Self { Self { action, dust_threshold, @@ -170,7 +175,7 @@ impl DustOutputPolicy { } /// Returns a value that will be used to override the dust determination logic of the /// change policy, if any. - pub fn dust_threshold(&self) -> Option { + pub fn dust_threshold(&self) -> Option { self.dust_threshold } } diff --git a/zcash_client_backend/src/fees/fixed.rs b/zcash_client_backend/src/fees/fixed.rs index 59732f8382..01d081e8a4 100644 --- a/zcash_client_backend/src/fees/fixed.rs +++ b/zcash_client_backend/src/fees/fixed.rs @@ -5,7 +5,7 @@ use zcash_primitives::{ consensus::{self, BlockHeight}, transaction::{ components::{ - amount::{Amount, BalanceError}, + amount::{Amount, BalanceError, NonNegativeAmount}, sapling::fees as sapling, transparent::fees as transparent, }, @@ -83,7 +83,9 @@ impl ChangeStrategy for SingleOutputChangeStrategy { ) .unwrap(); // fixed::FeeRule::fee_required is infallible. - let total_in = (t_in + sapling_in).ok_or(BalanceError::Overflow)?; + let total_in = (t_in + sapling_in) + .and_then(|v| NonNegativeAmount::try_from(v).ok()) + .ok_or(BalanceError::Overflow)?; if (!transparent_inputs.is_empty() || !sapling_inputs.is_empty()) && fee_amount > total_in { // For the fixed-fee selection rule, the only time we consider inputs dust is when the fee @@ -101,20 +103,22 @@ impl ChangeStrategy for SingleOutputChangeStrategy { .collect(), }) } else { - let total_out = [t_out, sapling_out, fee_amount] + let total_out = [t_out, sapling_out, fee_amount.into()] .iter() .sum::>() .ok_or(BalanceError::Overflow)?; - let proposed_change = (total_in - total_out).ok_or(BalanceError::Underflow)?; + let overflow = |_| ChangeError::StrategyError(BalanceError::Overflow); + let proposed_change = + (Amount::from(total_in) - total_out).ok_or(BalanceError::Underflow)?; match proposed_change.cmp(&Amount::zero()) { Ordering::Less => Err(ChangeError::InsufficientFunds { - available: total_in, + available: total_in.into(), required: total_out, }), - Ordering::Equal => TransactionBalance::new(vec![], fee_amount) - .ok_or_else(|| BalanceError::Overflow.into()), + Ordering::Equal => TransactionBalance::new(vec![], fee_amount).map_err(overflow), Ordering::Greater => { + let proposed_change = NonNegativeAmount::try_from(proposed_change).unwrap(); let dust_threshold = dust_output_policy .dust_threshold() .unwrap_or_else(|| self.fee_rule.fixed_fee()); @@ -125,28 +129,29 @@ impl ChangeStrategy for SingleOutputChangeStrategy { let shortfall = (dust_threshold - proposed_change) .ok_or(BalanceError::Underflow)?; Err(ChangeError::InsufficientFunds { - available: total_in, + available: total_in.into(), required: (total_in + shortfall) - .ok_or(BalanceError::Overflow)?, + .ok_or(BalanceError::Overflow)? + .into(), }) } DustAction::AllowDustChange => TransactionBalance::new( vec![ChangeValue::Sapling(proposed_change)], fee_amount, ) - .ok_or_else(|| BalanceError::Overflow.into()), + .map_err(overflow), DustAction::AddDustToFee => TransactionBalance::new( vec![], - (fee_amount + proposed_change).unwrap(), + (fee_amount + proposed_change).ok_or(BalanceError::Overflow)?, ) - .ok_or_else(|| BalanceError::Overflow.into()), + .map_err(overflow), } } else { TransactionBalance::new( vec![ChangeValue::Sapling(proposed_change)], fee_amount, ) - .ok_or_else(|| BalanceError::Overflow.into()) + .map_err(overflow) } } } @@ -159,7 +164,10 @@ mod tests { use zcash_primitives::{ consensus::{Network, NetworkUpgrade, Parameters}, transaction::{ - components::{amount::Amount, transparent::TxOut}, + components::{ + amount::{Amount, NonNegativeAmount}, + transparent::TxOut, + }, fees::fixed::FeeRule as FixedFeeRule, }, }; @@ -197,8 +205,8 @@ mod tests { assert_matches!( result, - Ok(balance) if balance.proposed_change() == [ChangeValue::Sapling(Amount::from_u64(10000).unwrap())] - && balance.fee_required() == Amount::from_u64(10000).unwrap() + Ok(balance) if balance.proposed_change() == [ChangeValue::Sapling(NonNegativeAmount::const_from_u64(10000))] + && balance.fee_required() == NonNegativeAmount::const_from_u64(10000) ); } diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index 6d2ddc62ab..1d17ae71d5 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -9,7 +9,7 @@ use zcash_primitives::{ consensus::{self, BlockHeight}, transaction::{ components::{ - amount::{Amount, BalanceError}, + amount::{Amount, BalanceError, NonNegativeAmount}, sapling::fees as sapling, transparent::fees as transparent, }, @@ -61,7 +61,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { .filter_map(|i| { // for now, we're just assuming p2pkh inputs, so we don't check the size of the input // script - if i.coin().value < self.fee_rule.marginal_fee() { + if i.coin().value < self.fee_rule.marginal_fee().into() { Some(i.outpoint().clone()) } else { None @@ -72,7 +72,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { let mut sapling_dust: Vec<_> = sapling_inputs .iter() .filter_map(|i| { - if i.value() < self.fee_rule.marginal_fee() { + if i.value() < self.fee_rule.marginal_fee().into() { Some(i.note_id().clone()) } else { None @@ -174,7 +174,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { let total_in = (t_in + sapling_in).ok_or_else(overflow)?; - let total_out = [t_out, sapling_out, fee_amount] + let total_out = [t_out, sapling_out, fee_amount.into()] .iter() .sum::>() .ok_or_else(overflow)?; @@ -185,8 +185,9 @@ impl ChangeStrategy for SingleOutputChangeStrategy { available: total_in, required: total_out, }), - Ordering::Equal => TransactionBalance::new(vec![], fee_amount).ok_or_else(overflow), + Ordering::Equal => TransactionBalance::new(vec![], fee_amount).map_err(|_| overflow()), Ordering::Greater => { + let proposed_change = NonNegativeAmount::try_from(proposed_change).unwrap(); let dust_threshold = dust_output_policy .dust_threshold() .unwrap_or_else(|| self.fee_rule.marginal_fee()); @@ -199,22 +200,23 @@ impl ChangeStrategy for SingleOutputChangeStrategy { Err(ChangeError::InsufficientFunds { available: total_in, - required: (total_in + shortfall).ok_or_else(overflow)?, + required: (total_in + shortfall.into()).ok_or_else(overflow)?, }) } DustAction::AllowDustChange => TransactionBalance::new( vec![ChangeValue::Sapling(proposed_change)], fee_amount, ) - .ok_or_else(overflow), - DustAction::AddDustToFee => { - TransactionBalance::new(vec![], (fee_amount + proposed_change).unwrap()) - .ok_or_else(overflow) - } + .map_err(|_| overflow()), + DustAction::AddDustToFee => TransactionBalance::new( + vec![], + (fee_amount + proposed_change).ok_or_else(overflow)?, + ) + .map_err(|_| overflow()), } } else { TransactionBalance::new(vec![ChangeValue::Sapling(proposed_change)], fee_amount) - .ok_or_else(overflow) + .map_err(|_| overflow()) } } } @@ -228,7 +230,10 @@ mod tests { consensus::{Network, NetworkUpgrade, Parameters}, legacy::Script, transaction::{ - components::{amount::Amount, transparent::TxOut}, + components::{ + amount::{Amount, NonNegativeAmount}, + transparent::TxOut, + }, fees::zip317::FeeRule as Zip317FeeRule, }, }; @@ -264,8 +269,8 @@ mod tests { assert_matches!( result, - Ok(balance) if balance.proposed_change() == [ChangeValue::Sapling(Amount::from_u64(5000).unwrap())] - && balance.fee_required() == Amount::from_u64(10000).unwrap() + Ok(balance) if balance.proposed_change() == [ChangeValue::Sapling(NonNegativeAmount::const_from_u64(5000))] + && balance.fee_required() == NonNegativeAmount::const_from_u64(10000) ); } @@ -295,7 +300,7 @@ mod tests { assert_matches!( result, Ok(balance) if balance.proposed_change().is_empty() - && balance.fee_required() == Amount::from_u64(15000).unwrap() + && balance.fee_required() == NonNegativeAmount::const_from_u64(15000) ); } @@ -328,7 +333,7 @@ mod tests { assert_matches!( result, Ok(balance) if balance.proposed_change().is_empty() - && balance.fee_required() == Amount::from_u64(10000).unwrap() + && balance.fee_required() == NonNegativeAmount::const_from_u64(10000) ); } diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index 919946040d..88597da91f 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -580,7 +580,7 @@ mod tests { value::NoteValue, Nullifier, SaplingIvk, }, - transaction::components::Amount, + transaction::components::amount::NonNegativeAmount, zip32::{AccountId, DiversifiableFullViewingKey, ExtendedSpendingKey}, }; @@ -637,7 +637,7 @@ mod tests { prev_hash: BlockHash, nf: Nullifier, dfvk: &DiversifiableFullViewingKey, - value: Amount, + value: NonNegativeAmount, tx_after: bool, initial_sapling_tree_size: Option, ) -> CompactBlock { @@ -646,7 +646,7 @@ mod tests { // Create a fake Note for the account let mut rng = OsRng; let rseed = generate_random_rseed(&Network::TestNetwork, height, &mut rng); - let note = sapling::Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); + let note = sapling::Note::from_parts(to, NoteValue::from(value), rseed); let encryptor = sapling_note_encryption::<_, Network>( Some(dfvk.fvk().ovk), note.clone(), @@ -724,7 +724,7 @@ mod tests { BlockHash([0; 32]), Nullifier([0; 32]), &dfvk, - Amount::from_u64(5).unwrap(), + NonNegativeAmount::const_from_u64(5), false, None, ); @@ -808,7 +808,7 @@ mod tests { BlockHash([0; 32]), Nullifier([0; 32]), &dfvk, - Amount::from_u64(5).unwrap(), + NonNegativeAmount::const_from_u64(5), true, Some(0), ); @@ -884,7 +884,7 @@ mod tests { BlockHash([0; 32]), nf, &dfvk, - Amount::from_u64(5).unwrap(), + NonNegativeAmount::const_from_u64(5), false, Some(0), ); diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index b7a7c63e04..43cb2ca9c6 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -367,7 +367,7 @@ mod tests { let (h1, _, _) = st.generate_next_block( &dfvk, AddressType::DefaultExternal, - Amount::from_u64(5).unwrap(), + NonNegativeAmount::const_from_u64(5), ); // Scan the cache @@ -377,7 +377,7 @@ mod tests { let (h2, _, _) = st.generate_next_block( &dfvk, AddressType::DefaultExternal, - Amount::from_u64(7).unwrap(), + NonNegativeAmount::const_from_u64(7), ); // Scanning should detect no inconsistencies @@ -397,12 +397,12 @@ mod tests { let (h, _, _) = st.generate_next_block( &dfvk, AddressType::DefaultExternal, - Amount::from_u64(5).unwrap(), + NonNegativeAmount::const_from_u64(5), ); let (last_contiguous_height, _, _) = st.generate_next_block( &dfvk, AddressType::DefaultExternal, - Amount::from_u64(7).unwrap(), + NonNegativeAmount::const_from_u64(7), ); // Scanning the cache should find no inconsistencies @@ -415,13 +415,13 @@ mod tests { BlockHash([1; 32]), &dfvk, AddressType::DefaultExternal, - Amount::from_u64(8).unwrap(), + NonNegativeAmount::const_from_u64(8), 2, ); st.generate_next_block( &dfvk, AddressType::DefaultExternal, - Amount::from_u64(3).unwrap(), + NonNegativeAmount::const_from_u64(3), ); // Data+cache chain should be invalid at the data/cache boundary @@ -448,10 +448,10 @@ mod tests { assert_eq!(st.get_wallet_summary(0), None); // Create fake CompactBlocks sending value to the address - let value = NonNegativeAmount::from_u64(5).unwrap(); - let value2 = NonNegativeAmount::from_u64(7).unwrap(); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); - st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2.into()); + let value = NonNegativeAmount::const_from_u64(5); + let value2 = NonNegativeAmount::const_from_u64(7); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); // Scan the cache st.scan_cached_blocks(h, 2); @@ -502,14 +502,14 @@ mod tests { let dfvk = st.test_account_sapling().unwrap(); // Create a block with height SAPLING_ACTIVATION_HEIGHT - let value = NonNegativeAmount::from_u64(50000).unwrap(); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + let value = NonNegativeAmount::const_from_u64(50000); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); st.scan_cached_blocks(h1, 1); assert_eq!(st.get_total_balance(AccountId::from(0)), value); // Create blocks to reach SAPLING_ACTIVATION_HEIGHT + 2 - let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); - let (h3, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let (h3, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); // Scan the later block first st.scan_cached_blocks(h3, 1); @@ -560,8 +560,8 @@ mod tests { assert_eq!(st.get_wallet_summary(0), None); // Create a fake CompactBlock sending value to the address - let value = NonNegativeAmount::from_u64(5).unwrap(); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + let value = NonNegativeAmount::const_from_u64(5); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); // Scan the cache let summary = st.scan_cached_blocks(h1, 1); @@ -573,8 +573,8 @@ mod tests { assert_eq!(st.get_total_balance(AccountId::from(0)), value); // Create a second fake CompactBlock sending more value to the address - let value2 = NonNegativeAmount::from_u64(7).unwrap(); - let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2.into()); + let value2 = NonNegativeAmount::const_from_u64(7); + let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); // Scan the cache again let summary = st.scan_cached_blocks(h2, 1); @@ -601,9 +601,9 @@ mod tests { assert_eq!(st.get_wallet_summary(0), None); // Create a fake CompactBlock sending value to the address - let value = NonNegativeAmount::from_u64(5).unwrap(); + let value = NonNegativeAmount::const_from_u64(5); let (received_height, _, nf) = - st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); // Scan the cache st.scan_cached_blocks(received_height, 1); @@ -614,9 +614,8 @@ mod tests { // Create a second fake CompactBlock spending value from the address let extsk2 = ExtendedSpendingKey::master(&[0]); let to2 = extsk2.default_address().1; - let value2 = NonNegativeAmount::from_u64(2).unwrap(); - let (spent_height, _) = - st.generate_next_block_spending(&dfvk, (nf, value.into()), to2, value2.into()); + let value2 = NonNegativeAmount::const_from_u64(2); + let (spent_height, _) = st.generate_next_block_spending(&dfvk, (nf, value), to2, value2); // Scan the cache again st.scan_cached_blocks(spent_height, 1); @@ -641,16 +640,15 @@ mod tests { assert_eq!(st.get_wallet_summary(0), None); // Create a fake CompactBlock sending value to the address - let value = NonNegativeAmount::from_u64(5).unwrap(); + let value = NonNegativeAmount::const_from_u64(5); let (received_height, _, nf) = - st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); // Create a second fake CompactBlock spending value from the address let extsk2 = ExtendedSpendingKey::master(&[0]); let to2 = extsk2.default_address().1; - let value2 = NonNegativeAmount::from_u64(2).unwrap(); - let (spent_height, _) = - st.generate_next_block_spending(&dfvk, (nf, value.into()), to2, value2.into()); + let value2 = NonNegativeAmount::const_from_u64(2); + let (spent_height, _) = st.generate_next_block_spending(&dfvk, (nf, value), to2, value2); // Scan the spending block first. st.scan_cached_blocks(spent_height, 1); diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 34dcf5ff42..6025838b5e 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -1135,7 +1135,9 @@ mod tests { use { crate::testing::AddressType, zcash_client_backend::keys::sapling, - zcash_primitives::{consensus::Parameters, transaction::components::Amount}, + zcash_primitives::{ + consensus::Parameters, transaction::components::amount::NonNegativeAmount, + }, }; #[test] @@ -1194,12 +1196,12 @@ mod tests { let (h1, meta1, _) = st.generate_next_block( &dfvk, AddressType::DefaultExternal, - Amount::from_u64(5).unwrap(), + NonNegativeAmount::const_from_u64(5), ); let (h2, meta2, _) = st.generate_next_block( &dfvk, AddressType::DefaultExternal, - Amount::from_u64(10).unwrap(), + NonNegativeAmount::const_from_u64(10), ); // The BlockMeta DB is not updated until we do so explicitly. diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index f6a9bbc7eb..a57b0199ff 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -48,10 +48,7 @@ use zcash_primitives::{ Note, Nullifier, PaymentAddress, }, transaction::{ - components::{ - amount::{BalanceError, NonNegativeAmount}, - Amount, - }, + components::amount::{BalanceError, NonNegativeAmount}, fees::FeeRule, Transaction, TxId, }, @@ -181,7 +178,7 @@ where &mut self, dfvk: &DiversifiableFullViewingKey, req: AddressType, - value: Amount, + value: NonNegativeAmount, ) -> (BlockHeight, Cache::InsertResult, Nullifier) { let (height, prev_hash, initial_sapling_tree_size) = self .latest_cached_block @@ -211,7 +208,7 @@ where prev_hash: BlockHash, dfvk: &DiversifiableFullViewingKey, req: AddressType, - value: Amount, + value: NonNegativeAmount, initial_sapling_tree_size: u32, ) -> (Cache::InsertResult, Nullifier) { let (cb, nf) = fake_compact_block( @@ -240,9 +237,9 @@ where pub(crate) fn generate_next_block_spending( &mut self, dfvk: &DiversifiableFullViewingKey, - note: (Nullifier, Amount), + note: (Nullifier, NonNegativeAmount), to: PaymentAddress, - value: Amount, + value: NonNegativeAmount, ) -> (BlockHeight, Cache::InsertResult) { let (height, prev_hash, initial_sapling_tree_size) = self .latest_cached_block @@ -434,7 +431,7 @@ impl TestState { &mut self, usk: &UnifiedSpendingKey, to: &RecipientAddress, - amount: Amount, + amount: NonNegativeAmount, memo: Option, ovk_policy: OvkPolicy, min_confirmations: NonZeroU32, @@ -700,7 +697,7 @@ pub(crate) fn fake_compact_block( prev_hash: BlockHash, dfvk: &DiversifiableFullViewingKey, req: AddressType, - value: Amount, + value: NonNegativeAmount, initial_sapling_tree_size: u32, ) -> (CompactBlock, Nullifier) { let to = match req { @@ -712,7 +709,7 @@ pub(crate) fn fake_compact_block( // Create a fake Note for the account let mut rng = OsRng; let rseed = generate_random_rseed(params, height, &mut rng); - let note = Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); + let note = Note::from_parts(to, NoteValue::from(value), rseed); let encryptor = sapling_note_encryption::<_, Network>( Some(dfvk.fvk().ovk), note.clone(), @@ -801,10 +798,10 @@ pub(crate) fn fake_compact_block_spending( params: &P, height: BlockHeight, prev_hash: BlockHash, - (nf, in_value): (Nullifier, Amount), + (nf, in_value): (Nullifier, NonNegativeAmount), dfvk: &DiversifiableFullViewingKey, to: PaymentAddress, - value: Amount, + value: NonNegativeAmount, initial_sapling_tree_size: u32, ) -> CompactBlock { let mut rng = OsRng; @@ -820,7 +817,7 @@ pub(crate) fn fake_compact_block_spending( // Create a fake Note for the payment ctx.outputs.push({ - let note = Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); + let note = Note::from_parts(to, NoteValue::from(value), rseed); let encryptor = sapling_note_encryption::<_, Network>( Some(dfvk.fvk().ovk), note.clone(), @@ -846,7 +843,7 @@ pub(crate) fn fake_compact_block_spending( let rseed = generate_random_rseed(params, height, &mut rng); let note = Note::from_parts( change_addr, - NoteValue::from_raw((in_value - value).unwrap().into()), + NoteValue::from((in_value - value).unwrap()), rseed, ); let encryptor = sapling_note_encryption::<_, Network>( diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 715d30f7c5..7656759e8c 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -2094,7 +2094,7 @@ mod tests { // Initialize the wallet with chain data that has no shielded notes for us. let not_our_key = ExtendedSpendingKey::master(&[]).to_diversifiable_full_viewing_key(); - let not_our_value = Amount::const_from_i64(10000); + let not_our_value = NonNegativeAmount::const_from_u64(10000); let (start_height, _, _) = st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); for _ in 1..10 { @@ -2160,7 +2160,9 @@ mod tests { // Shield the output. let input_selector = GreedyInputSelector::new( - fixed::SingleOutputChangeStrategy::new(FixedFeeRule::non_standard(Amount::zero())), + fixed::SingleOutputChangeStrategy::new(FixedFeeRule::non_standard( + NonNegativeAmount::ZERO, + )), DustOutputPolicy::default(), ); let txid = st diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index db5b6bb851..2b5f6775dc 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -514,8 +514,8 @@ pub(crate) mod tests { let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note - let value = NonNegativeAmount::from_u64(60000).unwrap(); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + let value = NonNegativeAmount::const_from_u64(60000); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); st.scan_cached_blocks(h, 1); // Spendable balance matches total balance @@ -534,7 +534,7 @@ pub(crate) mod tests { let to: RecipientAddress = to_extsk.default_address().1.into(); let request = zip321::TransactionRequest::new(vec![Payment { recipient_address: to, - amount: Amount::from_u64(10000).unwrap(), + amount: Amount::const_from_i64(10000), memo: None, // this should result in the creation of an empty memo label: None, message: None, @@ -666,7 +666,7 @@ pub(crate) mod tests { st.create_spend_to_address( &usk1, &to, - Amount::from_u64(1).unwrap(), + NonNegativeAmount::const_from_u64(1), None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), @@ -693,7 +693,7 @@ pub(crate) mod tests { st.create_spend_to_address( &usk, &to, - Amount::from_u64(1).unwrap(), + NonNegativeAmount::const_from_u64(1), None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), @@ -713,8 +713,8 @@ pub(crate) mod tests { let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note - let value = NonNegativeAmount::from_u64(50000).unwrap(); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + let value = NonNegativeAmount::const_from_u64(50000); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); st.scan_cached_blocks(h1, 1); // Spendable balance matches total balance at 1 confirmation. @@ -736,7 +736,7 @@ pub(crate) mod tests { ); // Add more funds to the wallet in a second note - let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); st.scan_cached_blocks(h2, 1); // Verified balance does not include the second note @@ -759,7 +759,7 @@ pub(crate) mod tests { st.create_spend_to_address( &usk, &to, - Amount::from_u64(70000).unwrap(), + NonNegativeAmount::const_from_u64(70000), None, OvkPolicy::Sender, NonZeroU32::new(10).unwrap(), @@ -768,14 +768,14 @@ pub(crate) mod tests { available, required }) - if available == Amount::from_u64(50000).unwrap() - && required == Amount::from_u64(80000).unwrap() + if available == Amount::const_from_i64(50000) + && required == Amount::const_from_i64(80000) ); // Mine blocks SAPLING_ACTIVATION_HEIGHT + 2 to 9 until just before the second // note is verified for _ in 2..10 { - st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); } st.scan_cached_blocks(h2 + 1, 8); @@ -787,7 +787,7 @@ pub(crate) mod tests { st.create_spend_to_address( &usk, &to, - Amount::from_u64(70000).unwrap(), + NonNegativeAmount::const_from_u64(70000), None, OvkPolicy::Sender, NonZeroU32::new(10).unwrap(), @@ -796,12 +796,12 @@ pub(crate) mod tests { available, required }) - if available == Amount::from_u64(50000).unwrap() - && required == Amount::from_u64(80000).unwrap() + if available == Amount::const_from_i64(50000) + && required == Amount::const_from_i64(80000) ); // Mine block 11 so that the second note becomes verified - let (h11, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + let (h11, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); st.scan_cached_blocks(h11, 1); // Total balance is value * number of blocks scanned (11). @@ -819,7 +819,7 @@ pub(crate) mod tests { .create_spend_to_address( &usk, &to, - amount_sent.into(), + amount_sent, None, OvkPolicy::Sender, NonZeroU32::new(10).unwrap(), @@ -849,8 +849,8 @@ pub(crate) mod tests { let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note - let value = NonNegativeAmount::from_u64(50000).unwrap(); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + let value = NonNegativeAmount::const_from_u64(50000); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); st.scan_cached_blocks(h1, 1); // Spendable balance matches total balance at 1 confirmation. @@ -864,7 +864,7 @@ pub(crate) mod tests { st.create_spend_to_address( &usk, &to, - Amount::from_u64(15000).unwrap(), + NonNegativeAmount::const_from_u64(15000), None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), @@ -877,7 +877,7 @@ pub(crate) mod tests { st.create_spend_to_address( &usk, &to, - Amount::from_u64(2000).unwrap(), + NonNegativeAmount::const_from_u64(2000), None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), @@ -886,7 +886,7 @@ pub(crate) mod tests { available, required }) - if available == Amount::zero() && required == Amount::from_u64(12000).unwrap() + if available == Amount::zero() && required == Amount::const_from_i64(12000) ); // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 41 (that don't send us funds) @@ -895,7 +895,7 @@ pub(crate) mod tests { st.generate_next_block( &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, - value.into(), + value, ); } st.scan_cached_blocks(h1 + 1, 41); @@ -905,7 +905,7 @@ pub(crate) mod tests { st.create_spend_to_address( &usk, &to, - Amount::from_u64(2000).unwrap(), + NonNegativeAmount::const_from_u64(2000), None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), @@ -914,14 +914,14 @@ pub(crate) mod tests { available, required }) - if available == Amount::zero() && required == Amount::from_u64(12000).unwrap() + if available == Amount::zero() && required == Amount::const_from_i64(12000) ); // Mine block SAPLING_ACTIVATION_HEIGHT + 42 so that the first transaction expires let (h43, _, _) = st.generate_next_block( &ExtendedSpendingKey::master(&[42]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, - value.into(), + value, ); st.scan_cached_blocks(h43, 1); @@ -935,7 +935,7 @@ pub(crate) mod tests { .create_spend_to_address( &usk, &to, - amount_sent2.into(), + amount_sent2, None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), @@ -964,8 +964,8 @@ pub(crate) mod tests { let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note - let value = NonNegativeAmount::from_u64(50000).unwrap(); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + let value = NonNegativeAmount::const_from_u64(50000); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); st.scan_cached_blocks(h1, 1); // Spendable balance matches total balance at 1 confirmation. @@ -991,7 +991,7 @@ pub(crate) mod tests { let txid = st.create_spend_to_address( &usk, &to, - Amount::from_u64(15000).unwrap(), + NonNegativeAmount::const_from_u64(15000), None, ovk_policy, NonZeroU32::new(1).unwrap(), @@ -1040,7 +1040,7 @@ pub(crate) mod tests { st.generate_next_block( &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, - value.into(), + value, ); } st.scan_cached_blocks(h1 + 1, 42); @@ -1064,8 +1064,8 @@ pub(crate) mod tests { let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note - let value = NonNegativeAmount::from_u64(60000).unwrap(); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + let value = NonNegativeAmount::const_from_u64(60000); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); st.scan_cached_blocks(h, 1); // Spendable balance matches total balance at 1 confirmation. @@ -1078,7 +1078,7 @@ pub(crate) mod tests { st.create_spend_to_address( &usk, &to, - Amount::from_u64(50000).unwrap(), + NonNegativeAmount::const_from_u64(50000), None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), @@ -1098,8 +1098,8 @@ pub(crate) mod tests { let dfvk = st.test_account_sapling().unwrap(); // Add funds to the wallet in a single note owned by the internal spending key - let value = NonNegativeAmount::from_u64(60000).unwrap(); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::Internal, value.into()); + let value = NonNegativeAmount::const_from_u64(60000); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::Internal, value); st.scan_cached_blocks(h, 1); // Spendable balance matches total balance at 1 confirmation. @@ -1119,7 +1119,7 @@ pub(crate) mod tests { st.create_spend_to_address( &usk, &to, - Amount::from_u64(50000).unwrap(), + NonNegativeAmount::const_from_u64(50000), None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), @@ -1149,7 +1149,7 @@ pub(crate) mod tests { // Add funds to the wallet in a single note let value = NonNegativeAmount::from_u64(100000).unwrap(); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value.into()); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); st.scan_cached_blocks(h, 1); // Spendable balance matches total balance @@ -1202,8 +1202,7 @@ pub(crate) mod tests { ) .unwrap(); - let amount_left = - (value - (amount_sent + fee_rule.fixed_fee().try_into().unwrap()).unwrap()).unwrap(); + let amount_left = (value - (amount_sent + fee_rule.fixed_fee()).unwrap()).unwrap(); let pending_change = (amount_left - amount_legacy_change).unwrap(); // The "legacy change" is not counted by get_pending_change(). @@ -1261,7 +1260,7 @@ pub(crate) mod tests { let (h1, _, _) = st.generate_next_block( &dfvk, AddressType::Internal, - Amount::from_u64(50000).unwrap(), + NonNegativeAmount::const_from_u64(50000), ); // Add 10 dust notes to the wallet @@ -1269,14 +1268,14 @@ pub(crate) mod tests { st.generate_next_block( &dfvk, AddressType::DefaultExternal, - Amount::from_u64(1000).unwrap(), + NonNegativeAmount::const_from_u64(1000), ); } st.scan_cached_blocks(h1, 11); // Spendable balance matches total balance - let total = NonNegativeAmount::from_u64(60000).unwrap(); + let total = NonNegativeAmount::const_from_u64(60000); assert_eq!(st.get_total_balance(account), total); assert_eq!(st.get_spendable_balance(account, 1), total); @@ -1288,7 +1287,7 @@ pub(crate) mod tests { // This first request will fail due to insufficient non-dust funds let req = TransactionRequest::new(vec![Payment { recipient_address: RecipientAddress::Shielded(dfvk.default_address().1), - amount: Amount::from_u64(50000).unwrap(), + amount: Amount::const_from_i64(50000), memo: None, label: None, message: None, @@ -1305,15 +1304,15 @@ pub(crate) mod tests { NonZeroU32::new(1).unwrap(), ), Err(Error::InsufficientFunds { available, required }) - if available == Amount::from_u64(51000).unwrap() - && required == Amount::from_u64(60000).unwrap() + if available == Amount::const_from_i64(51000) + && required == Amount::const_from_i64(60000) ); // This request will succeed, spending a single dust input to pay the 10000 // ZAT fee in addition to the 41000 ZAT output to the recipient let req = TransactionRequest::new(vec![Payment { recipient_address: RecipientAddress::Shielded(dfvk.default_address().1), - amount: Amount::from_u64(41000).unwrap(), + amount: Amount::const_from_i64(41000), memo: None, label: None, message: None, @@ -1365,14 +1364,14 @@ pub(crate) mod tests { let (h, _, _) = st.generate_next_block( &dfvk, AddressType::Internal, - Amount::from_u64(50000).unwrap(), + NonNegativeAmount::const_from_u64(50000), ); st.scan_cached_blocks(h, 1); let utxo = WalletTransparentOutput::from_parts( OutPoint::new([1u8; 32], 1), TxOut { - value: Amount::from_u64(10000).unwrap(), + value: Amount::const_from_i64(10000), script_pubkey: taddr.script(), }, h, @@ -1431,7 +1430,7 @@ pub(crate) mod tests { // Generate 9 blocks that have no value for us, starting at the birthday height. let not_our_key = ExtendedSpendingKey::master(&[]).to_diversifiable_full_viewing_key(); - let not_our_value = Amount::const_from_i64(10000); + let not_our_value = NonNegativeAmount::const_from_u64(10000); st.generate_block_at( birthday.height(), BlockHash([0; 32]), @@ -1448,7 +1447,7 @@ pub(crate) mod tests { st.generate_next_block( &dfvk, AddressType::DefaultExternal, - Amount::const_from_i64(500000), + NonNegativeAmount::const_from_u64(500000), ); // Generate some more blocks to get above our anchor height @@ -1502,14 +1501,14 @@ pub(crate) mod tests { st.generate_next_block( &dfvk, AddressType::DefaultExternal, - Amount::const_from_i64(500000), + NonNegativeAmount::const_from_u64(500000), ); st.scan_cached_blocks(birthday.height(), 1); // Create a gap of 10 blocks having no shielded outputs, then add a block that doesn't // belong to us so that we can get a checkpoint in the tree. let not_our_key = ExtendedSpendingKey::master(&[]).to_diversifiable_full_viewing_key(); - let not_our_value = Amount::const_from_i64(10000); + let not_our_value = NonNegativeAmount::const_from_u64(10000); st.generate_block_at( birthday.height() + 10, BlockHash([0; 32]), @@ -1545,7 +1544,7 @@ pub(crate) mod tests { st.create_spend_to_address( &usk, &to, - Amount::from_u64(10000).unwrap(), + NonNegativeAmount::const_from_u64(10000), None, OvkPolicy::Sender, NonZeroU32::new(5).unwrap(), diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 41d1e922f0..70a7c9d051 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -512,7 +512,7 @@ pub(crate) mod tests { block::BlockHash, consensus::{BlockHeight, NetworkUpgrade, Parameters}, sapling::Node, - transaction::components::Amount, + transaction::components::amount::NonNegativeAmount, zip32::DiversifiableFullViewingKey, }; @@ -564,7 +564,7 @@ pub(crate) mod tests { let initial_sapling_tree_size = (0x1 << 16) * 3 + 5; let initial_height = sapling_activation_height + 310; - let value = Amount::from_u64(50000).unwrap(); + let value = NonNegativeAmount::const_from_u64(50000); st.generate_block_at( initial_height, BlockHash([0; 32]), @@ -578,7 +578,7 @@ pub(crate) mod tests { st.generate_next_block( &dfvk, AddressType::DefaultExternal, - Amount::from_u64(10000).unwrap(), + NonNegativeAmount::const_from_u64(10000), ); } @@ -845,8 +845,8 @@ pub(crate) mod tests { BlockHash([0u8; 32]), &dfvk, AddressType::DefaultExternal, - Amount::const_from_i64(10000), // 1235 notes into into the second shard + NonNegativeAmount::const_from_u64(10000), u64::from(birthday.sapling_frontier().value().unwrap().position() + 1) .try_into() .unwrap(), @@ -961,7 +961,7 @@ pub(crate) mod tests { BlockHash([0u8; 32]), &dfvk, AddressType::DefaultExternal, - Amount::const_from_i64(10000), + NonNegativeAmount::const_from_u64(10000), u64::from(birthday.sapling_frontier().value().unwrap().position() + 1) .try_into() .unwrap(), diff --git a/zcash_extensions/src/transparent/demo.rs b/zcash_extensions/src/transparent/demo.rs index b5cd5e0be0..6935c8b72e 100644 --- a/zcash_extensions/src/transparent/demo.rs +++ b/zcash_extensions/src/transparent/demo.rs @@ -847,7 +847,7 @@ mod tests { let mut builder_b = demo_builder(tx_height + 1); let prevout_a = (OutPoint::new(tx_a.txid(), 0), tze_a.vout[0].clone()); - let value_xfr = (value - fee_rule.fixed_fee()).unwrap(); + let value_xfr = (value - fee_rule.fixed_fee().into()).unwrap(); builder_b .demo_transfer_to_close(prevout_a, value_xfr, preimage_1, h2) .map_err(|e| format!("transfer failure: {:?}", e)) @@ -873,7 +873,7 @@ mod tests { builder_c .add_transparent_output( &TransparentAddress::PublicKey([0; 20]), - (value_xfr - fee_rule.fixed_fee()).unwrap(), + (value_xfr - fee_rule.fixed_fee().into()).unwrap(), ) .unwrap(); diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index a1fb27e0ba..213f770938 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -21,16 +21,41 @@ and this library adheres to Rust's notion of set of closures. - Test helpers, behind the `test-dependencies` feature flag: - `zcash_primitives::prover::mock::{MockSpendProver, MockOutputProver}` -- MARGINAL_FEE pub const Amount: - - `zcash_primitives::transaction::fees::zip317::MARGINAL_FEE` +- Constants in `zcash_primitives::transaction::fees::zip317`: + - `MARGINAL_FEE` + - `GRACE_ACTIONS` + - `P2PKH_STANDARD_INPUT_SIZE` + - `P2PKH_STANDARD_OUTPUT_SIZE` +- `zcash_primitives::transaction::builder::get_fee` +- Additions related to `zcash_primitive::components::transaction::Amount` + and `zcash_primitive::components::transaction::NonNegativeAmount` + - `impl TryFrom for u64` + - `Amount::const_from_u64` + - `NonNegativeAmount::const_from_u64` + - `NonNegativeAmount::from_nonnegative_i64_le_bytes` + - `NonNegativeAmount::to_i64_le_bytes` + - `impl From<&NonNegativeAmount> for Amount` + - `impl From for u64` + - `impl From for zcash_primitives::sapling::value::NoteValue` + - `impl Sum for Option` + - `impl<'a> Sum<&'a NonNegativeAmount> for Option` ### Changed +- `zcash_primitives::transaction::fees`: + - `FeeRule::fee_required` now returns the required fee as a `NonNegativeAmount` instead + of as an `Amount`. + - When using the `zfuture` feature flag, `FutureFeeRule::fee_required_zfuture` now returns + the required fee as a `NonNegativeAmount` instead of as an `Amount`. + - `fees::fixed::FeeRule::fixed_fee` now wraps a `NonNegativeAmount` instead of an `Amount` + - `fees::zip317::FeeRule::marginal_fee` is now represented and exposed as a + `NonNegativeAmount` instead of an `Amount` - `zcash_primitives::transaction::components::sapling`: - `MapAuth` trait methods now take `&mut self` instead of `&self`. ### Removed - `zcash_primitives::constants`: - All `const` values (moved to `zcash_primitives::sapling::constants`). +- `impl From for u64` ## [0.13.0] - 2023-09-25 ### Added @@ -55,7 +80,7 @@ and this library adheres to Rust's notion of ### Changed - Migrated to `incrementalmerkletree 0.5`, `orchard 0.6`. - `zcash_primitives::transaction`: - - `builder::Builder::{new, new_with_rng}` now take an optional `orchard_anchor` + - `builder::Builder::{new, new_with_rng}` now take an optional `orchard_anchor` argument which must be provided in order to enable Orchard spends and recipients. - All `builder::Builder` methods now require the bound `R: CryptoRng` on `Builder<'a, P, R>`. A non-`CryptoRng` randomness source is still accepted @@ -64,11 +89,11 @@ and this library adheres to Rust's notion of - `builder::Error` has several additional variants for Orchard-related errors. - `fees::FeeRule::fee_required` now takes an additional argument, `orchard_action_count` - - `Unauthorized`'s associated type `OrchardAuth` is now + - `Unauthorized`'s associated type `OrchardAuth` is now `orchard::builder::InProgress` instead of `zcash_primitives::transaction::components::orchard::Unauthorized` - `zcash_primitives::consensus::NetworkUpgrade` now implements `PartialEq`, `Eq` -- `zcash_primitives::legacy::Script` now has a custom `Debug` implementation that +- `zcash_primitives::legacy::Script` now has a custom `Debug` implementation that renders script details in a much more legible fashion. - `zcash_primitives::sapling::redjubjub::Signature` now has a custom `Debug` implementation that renders details in a much more legible fashion. @@ -76,7 +101,7 @@ and this library adheres to Rust's notion of implementation that renders details in a much more legible fashion. ### Removed -- `impl {PartialEq, Eq} for transaction::builder::Error` +- `impl {PartialEq, Eq} for transaction::builder::Error` (use `assert_matches!` where error comparisons are required) - `zcash_primitives::transaction::components::orchard::Unauthorized` - `zcash_primitives::transaction::components::amount::DEFAULT_FEE` was diff --git a/zcash_primitives/src/sapling/note_encryption.rs b/zcash_primitives/src/sapling/note_encryption.rs index a12dd4404f..3e87b1458b 100644 --- a/zcash_primitives/src/sapling/note_encryption.rs +++ b/zcash_primitives/src/sapling/note_encryption.rs @@ -27,7 +27,7 @@ use crate::{ Diversifier, Note, PaymentAddress, Rseed, }, transaction::components::{ - amount::Amount, + amount::NonNegativeAmount, sapling::{self, OutputDescription}, }, }; @@ -82,9 +82,20 @@ where } // The unwraps below are guaranteed to succeed by the assertion above - let diversifier = Diversifier(plaintext[1..12].try_into().unwrap()); - let value = Amount::from_u64_le_bytes(plaintext[12..20].try_into().unwrap()).ok()?; - let r: [u8; 32] = plaintext[20..COMPACT_NOTE_SIZE].try_into().unwrap(); + let diversifier = Diversifier( + plaintext[1..12] + .try_into() + .expect("Note plaintext is checked to have length >= COMPACT_NOTE_SIZE."), + ); + let value = NonNegativeAmount::from_u64_le_bytes( + plaintext[12..20] + .try_into() + .expect("Note plaintext is checked to have length >= COMPACT_NOTE_SIZE."), + ) + .ok()?; + let r: [u8; 32] = plaintext[20..COMPACT_NOTE_SIZE] + .try_into() + .expect("Note plaintext is checked to have length >= COMPACT_NOTE_SIZE."); let rseed = if plaintext[0] == 0x01 { let rcm = Option::from(jubjub::Fr::from_repr(r))?; @@ -97,7 +108,7 @@ where // `diversifier` was checked by `get_pk_d`. let to = PaymentAddress::from_parts_unchecked(diversifier, pk_d)?; - let note = to.create_note(value.into(), rseed); + let note = to.create_note(value.try_into().unwrap(), rseed); Some((note, to)) } diff --git a/zcash_primitives/src/transaction/builder.rs b/zcash_primitives/src/transaction/builder.rs index d19d438f81..24e09f084e 100644 --- a/zcash_primitives/src/transaction/builder.rs +++ b/zcash_primitives/src/transaction/builder.rs @@ -45,6 +45,8 @@ use crate::{ }, }; +use super::components::amount::NonNegativeAmount; + /// Since Blossom activation, the default transaction expiry delta should be 40 blocks. /// const DEFAULT_TX_EXPIRY_DELTA: u32 = 40; @@ -345,7 +347,11 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { &mut self.rng, ovk, to, - NoteValue::from_raw(value.into()), + NoteValue::from_raw( + value + .try_into() + .expect("Cannot create Sapling outputs with negative note values."), + ), memo, ) } @@ -407,28 +413,26 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { /// /// This fee is a function of the spends and outputs that have been added to the builder, /// pursuant to the specified [`FeeRule`]. - pub fn get_fee(&self, fee_rule: &FR) -> Result> { - fee_rule - .fee_required( - &self.params, - self.target_height, - self.transparent_builder.inputs(), - self.transparent_builder.outputs(), - self.sapling_builder.inputs().len(), - self.sapling_builder.bundle_output_count(), - match std::cmp::max( - self.orchard_builder - .as_ref() - .map_or(0, |builder| builder.outputs().len()), - self.orchard_builder - .as_ref() - .map_or(0, |builder| builder.spends().len()), - ) { - 1 => 2, - n => n, - }, - ) - .map_err(Error::Fee) + pub fn get_fee(&self, fee_rule: &FR) -> Result { + fee_rule.fee_required( + &self.params, + self.target_height, + self.transparent_builder.inputs(), + self.transparent_builder.outputs(), + self.sapling_builder.inputs().len(), + self.sapling_builder.bundle_output_count(), + match std::cmp::max( + self.orchard_builder + .as_ref() + .map_or(0, |builder| builder.outputs().len()), + self.orchard_builder + .as_ref() + .map_or(0, |builder| builder.spends().len()), + ) { + 1 => 2, + n => n, + }, + ) } /// Builds a transaction from the configured spends and outputs. @@ -440,8 +444,8 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { prover: &impl TxProver, fee_rule: &FR, ) -> Result<(Transaction, SaplingMetadata), Error> { - let fee = self.get_fee(fee_rule)?; - self.build_internal(prover, fee) + let fee = self.get_fee(fee_rule).map_err(Error::Fee)?; + self.build_internal(prover, fee.into()) } /// Builds a transaction from the configured spends and outputs. @@ -467,7 +471,7 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { ) .map_err(Error::Fee)?; - self.build_internal(prover, fee) + self.build_internal(prover, fee.into()) } fn build_internal( @@ -714,7 +718,7 @@ mod tests { memo::MemoBytes, sapling::{Node, Rseed}, transaction::components::{ - amount::Amount, + amount::{Amount, NonNegativeAmount}, sapling::builder::{self as sapling_builder}, transparent::builder::{self as transparent_builder}, }, @@ -893,7 +897,7 @@ mod tests { let builder = Builder::new(TEST_NETWORK, tx_height, None); assert_matches!( builder.mock_build(), - Err(Error::InsufficientFunds(MINIMUM_FEE)) + Err(Error::InsufficientFunds(expected)) if expected == MINIMUM_FEE.into() ); } @@ -916,7 +920,7 @@ mod tests { assert_matches!( builder.mock_build(), Err(Error::InsufficientFunds(expected)) if - expected == (Amount::from_i64(50000).unwrap() + MINIMUM_FEE).unwrap() + expected == (NonNegativeAmount::const_from_u64(50000) + MINIMUM_FEE).unwrap().into() ); } @@ -933,7 +937,7 @@ mod tests { assert_matches!( builder.mock_build(), Err(Error::InsufficientFunds(expected)) if expected == - (Amount::from_i64(50000).unwrap() + MINIMUM_FEE).unwrap() + (NonNegativeAmount::const_from_u64(50000) + MINIMUM_FEE).unwrap().into() ); } @@ -971,7 +975,7 @@ mod tests { .unwrap(); assert_matches!( builder.mock_build(), - Err(Error::InsufficientFunds(expected)) if expected == Amount::from_i64(1).unwrap() + Err(Error::InsufficientFunds(expected)) if expected == Amount::const_from_i64(1) ); } diff --git a/zcash_primitives/src/transaction/components/amount.rs b/zcash_primitives/src/transaction/components/amount.rs index 4c03189629..996b2e7bc8 100644 --- a/zcash_primitives/src/transaction/components/amount.rs +++ b/zcash_primitives/src/transaction/components/amount.rs @@ -5,10 +5,12 @@ use std::ops::{Add, AddAssign, Mul, Neg, Sub, SubAssign}; use memuse::DynamicUsage; use orchard::value as orchard; +use crate::sapling::value::NoteValue; + pub const COIN: i64 = 1_0000_0000; pub const MAX_MONEY: i64 = 21_000_000 * COIN; -/// A type-safe representation of some quantity of Zcash. +/// A type-safe representation of a Zcash value delta, in zatoshis. /// /// An Amount can only be constructed from an integer that is within the valid monetary /// range of `{-MAX_MONEY..MAX_MONEY}` (where `MAX_MONEY` = 21,000,000 × 10⁸ zatoshis). @@ -38,6 +40,14 @@ impl Amount { Amount(amount) } + /// Creates a constant Amount from a u64. + /// + /// Panics: if the amount is outside the range `{0..MAX_MONEY}`. + const fn const_from_u64(amount: u64) -> Self { + assert!(amount <= (MAX_MONEY as u64)); // contains is not const + Amount(amount as i64) + } + /// Creates an Amount from an i64. /// /// Returns an error if the amount is outside the range `{-MAX_MONEY..MAX_MONEY}`. @@ -141,9 +151,11 @@ impl From<&Amount> for i64 { } } -impl From for u64 { - fn from(amount: Amount) -> u64 { - amount.0 as u64 +impl TryFrom for u64 { + type Error = (); + + fn try_from(value: Amount) -> Result { + value.0.try_into().map_err(|_| ()) } } @@ -248,12 +260,27 @@ impl NonNegativeAmount { Amount::from_u64(amount).map(NonNegativeAmount) } + /// Creates a constant NonNegativeAmount from a u64. + /// + /// Panics: if the amount is outside the range `{-MAX_MONEY..MAX_MONEY}`. + pub const fn const_from_u64(amount: u64) -> Self { + NonNegativeAmount(Amount::const_from_u64(amount)) + } + /// Creates a NonNegativeAmount from an i64. /// /// Returns an error if the amount is outside the range `{0..MAX_MONEY}`. pub fn from_nonnegative_i64(amount: i64) -> Result { Amount::from_nonnegative_i64(amount).map(NonNegativeAmount) } + + /// Reads an NonNegativeAmount from an unsigned 64-bit little-endian integer. + /// + /// Returns an error if the amount is outside the range `{0..MAX_MONEY}`. + pub fn from_u64_le_bytes(bytes: [u8; 8]) -> Result { + let amount = u64::from_le_bytes(bytes); + Self::from_u64(amount) + } } impl From for Amount { @@ -262,6 +289,24 @@ impl From for Amount { } } +impl From<&NonNegativeAmount> for Amount { + fn from(n: &NonNegativeAmount) -> Self { + n.0 + } +} + +impl From for u64 { + fn from(n: NonNegativeAmount) -> Self { + n.0.try_into().unwrap() + } +} + +impl From for NoteValue { + fn from(n: NonNegativeAmount) -> Self { + NoteValue::from_raw(n.0.try_into().unwrap()) + } +} + impl TryFrom for NonNegativeAmount { type Error = (); @@ -310,7 +355,19 @@ impl Mul for NonNegativeAmount { type Output = Option; fn mul(self, rhs: usize) -> Option { - (self.0 * rhs).map(NonNegativeAmount) + (self.0 * rhs).and_then(|v| NonNegativeAmount::try_from(v).ok()) + } +} + +impl Sum for Option { + fn sum>(iter: I) -> Self { + iter.fold(Some(NonNegativeAmount::ZERO), |acc, a| acc? + a) + } +} + +impl<'a> Sum<&'a NonNegativeAmount> for Option { + fn sum>(iter: I) -> Self { + iter.fold(Some(NonNegativeAmount::ZERO), |acc, a| acc? + *a) } } diff --git a/zcash_primitives/src/transaction/fees.rs b/zcash_primitives/src/transaction/fees.rs index 515c74f272..9ca89b9c66 100644 --- a/zcash_primitives/src/transaction/fees.rs +++ b/zcash_primitives/src/transaction/fees.rs @@ -2,7 +2,7 @@ use crate::{ consensus::{self, BlockHeight}, - transaction::components::{amount::Amount, transparent::fees as transparent}, + transaction::components::{amount::NonNegativeAmount, transparent::fees as transparent}, }; #[cfg(feature = "zfuture")] @@ -31,7 +31,7 @@ pub trait FeeRule { sapling_input_count: usize, sapling_output_count: usize, orchard_action_count: usize, - ) -> Result; + ) -> Result; } /// A trait that represents the ability to compute the fees that must be paid by a transaction @@ -54,5 +54,5 @@ pub trait FutureFeeRule: FeeRule { sapling_output_count: usize, tze_inputs: &[impl tze::InputView], tze_outputs: &[impl tze::OutputView], - ) -> Result; + ) -> Result; } diff --git a/zcash_primitives/src/transaction/fees/fixed.rs b/zcash_primitives/src/transaction/fees/fixed.rs index 0b2278a544..bb323a3867 100644 --- a/zcash_primitives/src/transaction/fees/fixed.rs +++ b/zcash_primitives/src/transaction/fees/fixed.rs @@ -1,6 +1,6 @@ use crate::{ consensus::{self, BlockHeight}, - transaction::components::{amount::Amount, transparent::fees as transparent}, + transaction::components::{amount::NonNegativeAmount, transparent::fees as transparent}, transaction::fees::zip317, }; @@ -11,12 +11,12 @@ use crate::transaction::components::tze::fees as tze; /// the transaction being constructed. #[derive(Clone, Copy, Debug)] pub struct FeeRule { - fixed_fee: Amount, + fixed_fee: NonNegativeAmount, } impl FeeRule { /// Creates a new nonstandard fixed fee rule with the specified fixed fee. - pub fn non_standard(fixed_fee: Amount) -> Self { + pub fn non_standard(fixed_fee: NonNegativeAmount) -> Self { Self { fixed_fee } } @@ -40,7 +40,7 @@ impl FeeRule { } /// Returns the fixed fee amount which which this rule was configured. - pub fn fixed_fee(&self) -> Amount { + pub fn fixed_fee(&self) -> NonNegativeAmount { self.fixed_fee } } @@ -57,7 +57,7 @@ impl super::FeeRule for FeeRule { _sapling_input_count: usize, _sapling_output_count: usize, _orchard_action_count: usize, - ) -> Result { + ) -> Result { Ok(self.fixed_fee) } } @@ -74,7 +74,7 @@ impl super::FutureFeeRule for FeeRule { _sapling_output_count: usize, _tze_inputs: &[impl tze::InputView], _tze_outputs: &[impl tze::OutputView], - ) -> Result { + ) -> Result { Ok(self.fixed_fee) } } diff --git a/zcash_primitives/src/transaction/fees/zip317.rs b/zcash_primitives/src/transaction/fees/zip317.rs index d1bb5efd8b..b0b08d7c03 100644 --- a/zcash_primitives/src/transaction/fees/zip317.rs +++ b/zcash_primitives/src/transaction/fees/zip317.rs @@ -8,22 +8,36 @@ use crate::{ consensus::{self, BlockHeight}, legacy::TransparentAddress, transaction::components::{ - amount::{Amount, BalanceError}, + amount::{BalanceError, NonNegativeAmount}, transparent::{fees as transparent, OutPoint}, }, }; -/// The minimum conventional fee using the standard [ZIP 317] constants. Equivalent to -/// `(FeeRule::standard().marginal_fee() * FeeRule::standard().grace_actions()).unwrap()`, -/// but as a constant. +/// The standard [ZIP 317] marginal fee. /// /// [ZIP 317]: https//zips.z.cash/zip-0317 -pub const MINIMUM_FEE: Amount = Amount::const_from_i64(10_000); +pub const MARGINAL_FEE: NonNegativeAmount = NonNegativeAmount::const_from_u64(5_000); -/// The marginal fee using the standard [ZIP 317] constants. +/// The minimum number of logical actions that must be paid according to [ZIP 317]. /// /// [ZIP 317]: https//zips.z.cash/zip-0317 -pub const MARGINAL_FEE: Amount = Amount::const_from_i64(5_000); +pub const GRACE_ACTIONS: usize = 2; + +/// The standard size of a P2PKH input, in bytes, according to [ZIP 317]. +/// +/// [ZIP 317]: https//zips.z.cash/zip-0317 +pub const P2PKH_STANDARD_INPUT_SIZE: usize = 150; + +/// The standard size of a P2PKH output, in bytes, according to [ZIP 317]. +/// +/// [ZIP 317]: https//zips.z.cash/zip-0317 +pub const P2PKH_STANDARD_OUTPUT_SIZE: usize = 34; + +/// The minimum conventional fee computed from the standard [ZIP 317] constants. Equivalent to +/// `MARGINAL_FEE * GRACE_ACTIONS`. +/// +/// [ZIP 317]: https//zips.z.cash/zip-0317 +pub const MINIMUM_FEE: NonNegativeAmount = NonNegativeAmount::const_from_u64(10_000); /// A [`FeeRule`] implementation that implements the [ZIP 317] fee rule. /// @@ -35,7 +49,7 @@ pub const MARGINAL_FEE: Amount = Amount::const_from_i64(5_000); /// [ZIP 317]: https//zips.z.cash/zip-0317 #[derive(Clone, Debug)] pub struct FeeRule { - marginal_fee: Amount, + marginal_fee: NonNegativeAmount, grace_actions: usize, p2pkh_standard_input_size: usize, p2pkh_standard_output_size: usize, @@ -48,9 +62,9 @@ impl FeeRule { pub fn standard() -> Self { Self { marginal_fee: MARGINAL_FEE, - grace_actions: 2, - p2pkh_standard_input_size: 150, - p2pkh_standard_output_size: 34, + grace_actions: GRACE_ACTIONS, + p2pkh_standard_input_size: P2PKH_STANDARD_INPUT_SIZE, + p2pkh_standard_output_size: P2PKH_STANDARD_OUTPUT_SIZE, } } @@ -59,7 +73,7 @@ impl FeeRule { /// Returns `None` if either `p2pkh_standard_input_size` or `p2pkh_standard_output_size` are /// zero. pub fn non_standard( - marginal_fee: Amount, + marginal_fee: NonNegativeAmount, grace_actions: usize, p2pkh_standard_input_size: usize, p2pkh_standard_output_size: usize, @@ -77,7 +91,7 @@ impl FeeRule { } /// Returns the ZIP 317 marginal fee. - pub fn marginal_fee(&self) -> Amount { + pub fn marginal_fee(&self) -> NonNegativeAmount { self.marginal_fee } /// Returns the ZIP 317 number of grace actions @@ -135,7 +149,7 @@ impl super::FeeRule for FeeRule { sapling_input_count: usize, sapling_output_count: usize, orchard_action_count: usize, - ) -> Result { + ) -> Result { let non_p2pkh_inputs: Vec<_> = transparent_inputs .iter() .filter_map(|t_in| match t_in.coin().script_pubkey.address() { From 1447d8ea0140fa8ef3a087338a1d7bf3f1a6c8af Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 1 Aug 2023 15:03:24 -0600 Subject: [PATCH 0246/1122] zcash_client_backend: Move change memos into the `ChangeValue` components of `Proposal`s. The existing API limited change outputs to having only a single memo repeated across each change output. This change makes it so that each proposed change output can have its own associated memo, and leaves it up to the input selector to determine how requested change memos are associated with change outputs. --- zcash_client_backend/CHANGELOG.md | 17 ++++++++++ zcash_client_backend/src/data_api/wallet.rs | 21 ++++-------- zcash_client_backend/src/fees.rs | 20 +++++++++-- zcash_client_backend/src/fees/fixed.rs | 28 +++++++++++----- zcash_client_backend/src/fees/zip317.rs | 37 +++++++++++++++------ zcash_client_sqlite/src/chain.rs | 2 +- zcash_client_sqlite/src/testing.rs | 7 ++-- zcash_client_sqlite/src/wallet.rs | 9 +++-- zcash_client_sqlite/src/wallet/sapling.rs | 26 +++++++++++---- 9 files changed, 114 insertions(+), 53 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index d734954ec2..0011d77b5c 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -26,6 +26,23 @@ and this library adheres to Rust's notion of an `Amount`. - All uses of `Amount` in `data_api::wallet::input_selection` have been replaced with `NonNegativeAmount`. + - `wallet::shield_transparent_funds` no longer + takes a `memo` argument; instead, memos to be associated with the shielded + outputs should be specified in the construction of the value of the + `input_selector` argument, which is used to construct the proposed shielded + values as internal "change" outputs. + - `wallet::create_proposed_transaction` no longer takes a + `change_memo` argument; instead, change memos are represented in the + individual values of the `proposed_change` field of the `Proposal`'s + `TransactionBalance`. + - `wallet::create_spend_to_address` now takes an additional + `change_memo` argument. +- `zcash_client_backend::fees::ChangeValue::Sapling` is now a structured variant. + In addition to the existing change value, it now also carries an optional memo + to be associated with the change output. +- `zcash_client_backend::fees::fixed::SingleOutputChangeStrategy::new` and + `zcash_client_backend::fees::zip317::SingleOutputChangeStrategy::new` each now + accept an additional `change_memo` argument. - All uses of `Amount` in `zcash_client_backend::fees` have been replaced with `NonNegativeAmount`. diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index ddc5b5c38d..a6a34a6d70 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -196,6 +196,7 @@ pub fn create_spend_to_address( memo: Option, ovk_policy: OvkPolicy, min_confirmations: NonZeroU32, + change_memo: Option, ) -> Result< TxId, Error< @@ -224,7 +225,7 @@ where #[allow(deprecated)] let fee_rule = fixed::FeeRule::standard(); - let change_strategy = fees::fixed::SingleOutputChangeStrategy::new(fee_rule); + let change_strategy = fees::fixed::SingleOutputChangeStrategy::new(fee_rule, change_memo); spend( wallet_db, params, @@ -335,7 +336,6 @@ where ovk_policy, proposal, min_confirmations, - None, ) } @@ -418,7 +418,6 @@ pub fn create_proposed_transaction( ovk_policy: OvkPolicy, proposal: Proposal, min_confirmations: NonZeroU32, - change_memo: Option, ) -> Result< TxId, Error< @@ -566,14 +565,12 @@ where for change_value in proposal.balance().proposed_change() { match change_value { - ChangeValue::Sapling(amount) => { - let memo = change_memo - .as_ref() - .map_or_else(MemoBytes::empty, |m| m.clone()); + ChangeValue::Sapling { value, memo } => { + let memo = memo.as_ref().map_or_else(MemoBytes::empty, |m| m.clone()); builder.add_sapling_output( internal_ovk(), dfvk.change_address().1, - amount.into(), + value.into(), memo.clone(), )?; sapling_output_meta.push(( @@ -581,7 +578,7 @@ where account, PoolType::Shielded(ShieldedProtocol::Sapling), ), - amount.into(), + value.into(), Some(memo), )) } @@ -682,10 +679,6 @@ where /// * `from_addrs`: The list of transparent addresses that will be used to filter transaparent /// UTXOs received by the wallet. Only UTXOs received at one of the provided addresses will /// be selected to be shielded. -/// * `memo`: A memo to be included in the output to the (internal) recipient. -/// This can be used to take notes about auto-shielding operations internal -/// to the wallet that the wallet can use to improve how it represents those -/// shielding transactions to the user. /// * `min_confirmations`: The minimum number of confirmations that a previously /// received note must have in the blockchain in order to be considered for being /// spent. A value of 10 confirmations is recommended and 0-conf transactions are @@ -703,7 +696,6 @@ pub fn shield_transparent_funds( shielding_threshold: NonNegativeAmount, usk: &UnifiedSpendingKey, from_addrs: &[TransparentAddress], - memo: &MemoBytes, min_confirmations: NonZeroU32, ) -> Result< TxId, @@ -737,7 +729,6 @@ where OvkPolicy::Sender, proposal, min_confirmations, - Some(memo.clone()), ) } diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index 1681c225b2..1698c30c02 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -2,6 +2,7 @@ use std::fmt; use zcash_primitives::{ consensus::{self, BlockHeight}, + memo::MemoBytes, transaction::{ components::{ amount::{Amount, BalanceError, NonNegativeAmount}, @@ -19,13 +20,28 @@ pub mod zip317; /// A proposed change amount and output pool. #[derive(Clone, Debug, PartialEq, Eq)] pub enum ChangeValue { - Sapling(NonNegativeAmount), + Sapling { + value: NonNegativeAmount, + memo: Option, + }, } impl ChangeValue { + pub fn sapling(value: NonNegativeAmount, memo: Option) -> Self { + Self::Sapling { value, memo } + } + + /// Returns the value of the change output to be created, in zatoshis. pub fn value(&self) -> NonNegativeAmount { match self { - ChangeValue::Sapling(value) => *value, + ChangeValue::Sapling { value, .. } => *value, + } + } + + /// Returns the memo to be associated with the change output. + pub fn memo(&self) -> Option<&MemoBytes> { + match self { + ChangeValue::Sapling { memo, .. } => memo.as_ref(), } } } diff --git a/zcash_client_backend/src/fees/fixed.rs b/zcash_client_backend/src/fees/fixed.rs index 01d081e8a4..f4598c8249 100644 --- a/zcash_client_backend/src/fees/fixed.rs +++ b/zcash_client_backend/src/fees/fixed.rs @@ -3,6 +3,7 @@ use std::cmp::Ordering; use zcash_primitives::{ consensus::{self, BlockHeight}, + memo::MemoBytes, transaction::{ components::{ amount::{Amount, BalanceError, NonNegativeAmount}, @@ -21,12 +22,16 @@ use super::{ /// shielded pool and delegates fee calculation to the provided fee rule. pub struct SingleOutputChangeStrategy { fee_rule: FixedFeeRule, + change_memo: Option, } impl SingleOutputChangeStrategy { /// Constructs a new [`SingleOutputChangeStrategy`] with the specified fee rule. - pub fn new(fee_rule: FixedFeeRule) -> Self { - Self { fee_rule } + pub fn new(fee_rule: FixedFeeRule, change_memo: Option) -> Self { + Self { + fee_rule, + change_memo, + } } } @@ -136,7 +141,10 @@ impl ChangeStrategy for SingleOutputChangeStrategy { }) } DustAction::AllowDustChange => TransactionBalance::new( - vec![ChangeValue::Sapling(proposed_change)], + vec![ChangeValue::sapling( + proposed_change, + self.change_memo.clone(), + )], fee_amount, ) .map_err(overflow), @@ -148,7 +156,10 @@ impl ChangeStrategy for SingleOutputChangeStrategy { } } else { TransactionBalance::new( - vec![ChangeValue::Sapling(proposed_change)], + vec![ChangeValue::sapling( + proposed_change, + self.change_memo.clone(), + )], fee_amount, ) .map_err(overflow) @@ -185,7 +196,7 @@ mod tests { fn change_without_dust() { #[allow(deprecated)] let fee_rule = FixedFeeRule::standard(); - let change_strategy = SingleOutputChangeStrategy::new(fee_rule); + let change_strategy = SingleOutputChangeStrategy::new(fee_rule, None); // spend a single Sapling note that is sufficient to pay the fee let result = change_strategy.compute_balance( @@ -205,8 +216,9 @@ mod tests { assert_matches!( result, - Ok(balance) if balance.proposed_change() == [ChangeValue::Sapling(NonNegativeAmount::const_from_u64(10000))] - && balance.fee_required() == NonNegativeAmount::const_from_u64(10000) + Ok(balance) if + balance.proposed_change() == [ChangeValue::sapling(NonNegativeAmount::const_from_u64(10000), None)] && + balance.fee_required() == NonNegativeAmount::const_from_u64(10000) ); } @@ -214,7 +226,7 @@ mod tests { fn dust_change() { #[allow(deprecated)] let fee_rule = FixedFeeRule::standard(); - let change_strategy = SingleOutputChangeStrategy::new(fee_rule); + let change_strategy = SingleOutputChangeStrategy::new(fee_rule, None); // spend a single Sapling note that is sufficient to pay the fee let result = change_strategy.compute_balance( diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index 1d17ae71d5..4c1f7d6f65 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -7,6 +7,7 @@ use core::cmp::Ordering; use zcash_primitives::{ consensus::{self, BlockHeight}, + memo::MemoBytes, transaction::{ components::{ amount::{Amount, BalanceError, NonNegativeAmount}, @@ -28,13 +29,17 @@ use super::{ /// shielded pool and delegates fee calculation to the provided fee rule. pub struct SingleOutputChangeStrategy { fee_rule: Zip317FeeRule, + change_memo: Option, } impl SingleOutputChangeStrategy { /// Constructs a new [`SingleOutputChangeStrategy`] with the specified ZIP 317 /// fee parameters. - pub fn new(fee_rule: Zip317FeeRule) -> Self { - Self { fee_rule } + pub fn new(fee_rule: Zip317FeeRule, change_memo: Option) -> Self { + Self { + fee_rule, + change_memo, + } } } @@ -204,7 +209,10 @@ impl ChangeStrategy for SingleOutputChangeStrategy { }) } DustAction::AllowDustChange => TransactionBalance::new( - vec![ChangeValue::Sapling(proposed_change)], + vec![ChangeValue::sapling( + proposed_change, + self.change_memo.clone(), + )], fee_amount, ) .map_err(|_| overflow()), @@ -215,8 +223,14 @@ impl ChangeStrategy for SingleOutputChangeStrategy { .map_err(|_| overflow()), } } else { - TransactionBalance::new(vec![ChangeValue::Sapling(proposed_change)], fee_amount) - .map_err(|_| overflow()) + TransactionBalance::new( + vec![ChangeValue::sapling( + proposed_change, + self.change_memo.clone(), + )], + fee_amount, + ) + .map_err(|_| overflow()) } } } @@ -249,7 +263,7 @@ mod tests { #[test] fn change_without_dust() { - let change_strategy = SingleOutputChangeStrategy::new(Zip317FeeRule::standard()); + let change_strategy = SingleOutputChangeStrategy::new(Zip317FeeRule::standard(), None); // spend a single Sapling note that is sufficient to pay the fee let result = change_strategy.compute_balance( @@ -269,14 +283,15 @@ mod tests { assert_matches!( result, - Ok(balance) if balance.proposed_change() == [ChangeValue::Sapling(NonNegativeAmount::const_from_u64(5000))] - && balance.fee_required() == NonNegativeAmount::const_from_u64(10000) + Ok(balance) if + balance.proposed_change() == [ChangeValue::sapling(NonNegativeAmount::const_from_u64(5000), None)] && + balance.fee_required() == NonNegativeAmount::const_from_u64(10000) ); } #[test] fn change_with_transparent_payments() { - let change_strategy = SingleOutputChangeStrategy::new(Zip317FeeRule::standard()); + let change_strategy = SingleOutputChangeStrategy::new(Zip317FeeRule::standard(), None); // spend a single Sapling note that is sufficient to pay the fee let result = change_strategy.compute_balance( @@ -306,7 +321,7 @@ mod tests { #[test] fn change_with_allowable_dust() { - let change_strategy = SingleOutputChangeStrategy::new(Zip317FeeRule::standard()); + let change_strategy = SingleOutputChangeStrategy::new(Zip317FeeRule::standard(), None); // spend a single Sapling note that is sufficient to pay the fee let result = change_strategy.compute_balance( @@ -339,7 +354,7 @@ mod tests { #[test] fn change_with_disallowed_dust() { - let change_strategy = SingleOutputChangeStrategy::new(Zip317FeeRule::standard()); + let change_strategy = SingleOutputChangeStrategy::new(Zip317FeeRule::standard(), None); // spend a single Sapling note that is sufficient to pay the fee let result = change_strategy.compute_balance( diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 43cb2ca9c6..9756745658 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -532,7 +532,7 @@ mod tests { }]) .unwrap(); let input_selector = GreedyInputSelector::new( - SingleOutputChangeStrategy::new(FeeRule::standard()), + SingleOutputChangeStrategy::new(FeeRule::standard(), None), DustOutputPolicy::default(), ); assert_matches!( diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index a57b0199ff..49dd50a8db 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -427,6 +427,7 @@ impl TestState { /// Invokes [`create_spend_to_address`] with the given arguments. #[allow(deprecated)] #[allow(clippy::type_complexity)] + #[allow(clippy::too_many_arguments)] pub(crate) fn create_spend_to_address( &mut self, usk: &UnifiedSpendingKey, @@ -435,6 +436,7 @@ impl TestState { memo: Option, ovk_policy: OvkPolicy, min_confirmations: NonZeroU32, + change_memo: Option, ) -> Result< TxId, data_api::error::Error< @@ -455,6 +457,7 @@ impl TestState { memo, ovk_policy, min_confirmations, + change_memo, ) } @@ -563,7 +566,6 @@ impl TestState { ovk_policy: OvkPolicy, proposal: Proposal, min_confirmations: NonZeroU32, - change_memo: Option, ) -> Result< TxId, data_api::error::Error< @@ -585,7 +587,6 @@ impl TestState { ovk_policy, proposal, min_confirmations, - change_memo, ) } @@ -598,7 +599,6 @@ impl TestState { shielding_threshold: NonNegativeAmount, usk: &UnifiedSpendingKey, from_addrs: &[TransparentAddress], - memo: &MemoBytes, min_confirmations: NonZeroU32, ) -> Result< TxId, @@ -621,7 +621,6 @@ impl TestState { shielding_threshold, usk, from_addrs, - memo, min_confirmations, ) } diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 7656759e8c..12a762609c 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1947,7 +1947,6 @@ mod tests { }, zcash_primitives::{ consensus::BlockHeight, - memo::MemoBytes, transaction::{ components::{amount::NonNegativeAmount, Amount, OutPoint, TxOut}, fees::fixed::FeeRule as FixedFeeRule, @@ -2160,9 +2159,10 @@ mod tests { // Shield the output. let input_selector = GreedyInputSelector::new( - fixed::SingleOutputChangeStrategy::new(FixedFeeRule::non_standard( - NonNegativeAmount::ZERO, - )), + fixed::SingleOutputChangeStrategy::new( + FixedFeeRule::non_standard(NonNegativeAmount::ZERO), + None, + ), DustOutputPolicy::default(), ); let txid = st @@ -2171,7 +2171,6 @@ mod tests { value, &usk, &[*taddr], - &MemoBytes::empty(), NonZeroU32::new(1).unwrap(), ) .unwrap(); diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 2b5f6775dc..6072bb48cd 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -543,7 +543,9 @@ pub(crate) mod tests { .unwrap(); let fee_rule = FixedFeeRule::standard(); - let change_strategy = fixed::SingleOutputChangeStrategy::new(fee_rule); + let change_memo = "Test change memo".parse::().unwrap(); + let change_strategy = + fixed::SingleOutputChangeStrategy::new(fee_rule, Some(change_memo.clone().into())); let input_selector = &GreedyInputSelector::new(change_strategy, DustOutputPolicy::default()); let proposal_result = st.propose_transfer( @@ -554,13 +556,11 @@ pub(crate) mod tests { ); assert_matches!(proposal_result, Ok(_)); - let change_memo = "Test change memo".parse::().unwrap(); let create_proposed_result = st.create_proposed_transaction( &usk, OvkPolicy::Sender, proposal_result.unwrap(), NonZeroU32::new(1).unwrap(), - Some(change_memo.clone().into()), ); assert_matches!(create_proposed_result, Ok(_)); @@ -670,6 +670,7 @@ pub(crate) mod tests { None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), + None ), Err(data_api::error::Error::KeyNotRecognized) ); @@ -697,6 +698,7 @@ pub(crate) mod tests { None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), + None ), Err(data_api::error::Error::ScanRequired) ); @@ -763,6 +765,7 @@ pub(crate) mod tests { None, OvkPolicy::Sender, NonZeroU32::new(10).unwrap(), + None ), Err(data_api::error::Error::InsufficientFunds { available, @@ -791,6 +794,7 @@ pub(crate) mod tests { None, OvkPolicy::Sender, NonZeroU32::new(10).unwrap(), + None ), Err(data_api::error::Error::InsufficientFunds { available, @@ -823,6 +827,7 @@ pub(crate) mod tests { None, OvkPolicy::Sender, NonZeroU32::new(10).unwrap(), + None, ) .unwrap(); @@ -868,6 +873,7 @@ pub(crate) mod tests { None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), + None ), Ok(_) ); @@ -881,6 +887,7 @@ pub(crate) mod tests { None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), + None ), Err(data_api::error::Error::InsufficientFunds { available, @@ -909,6 +916,7 @@ pub(crate) mod tests { None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), + None ), Err(data_api::error::Error::InsufficientFunds { available, @@ -939,6 +947,7 @@ pub(crate) mod tests { None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), + None, ) .unwrap(); @@ -995,6 +1004,7 @@ pub(crate) mod tests { None, ovk_policy, NonZeroU32::new(1).unwrap(), + None, )?; // Fetch the transaction from the database @@ -1082,6 +1092,7 @@ pub(crate) mod tests { None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), + None ), Ok(_) ); @@ -1123,6 +1134,7 @@ pub(crate) mod tests { None, OvkPolicy::Sender, NonZeroU32::new(1).unwrap(), + None ), Ok(_) ); @@ -1188,7 +1200,7 @@ pub(crate) mod tests { let fee_rule = FixedFeeRule::standard(); let input_selector = GreedyInputSelector::new( - fixed::SingleOutputChangeStrategy::new(fee_rule), + fixed::SingleOutputChangeStrategy::new(fee_rule, None), DustOutputPolicy::default(), ); @@ -1280,7 +1292,7 @@ pub(crate) mod tests { assert_eq!(st.get_spendable_balance(account, 1), total); let input_selector = GreedyInputSelector::new( - zip317::SingleOutputChangeStrategy::new(Zip317FeeRule::standard()), + zip317::SingleOutputChangeStrategy::new(Zip317FeeRule::standard(), None), DustOutputPolicy::default(), ); @@ -1382,7 +1394,7 @@ pub(crate) mod tests { assert!(matches!(res0, Ok(_))); let input_selector = GreedyInputSelector::new( - fixed::SingleOutputChangeStrategy::new(FixedFeeRule::standard()), + fixed::SingleOutputChangeStrategy::new(FixedFeeRule::standard(), None), DustOutputPolicy::default(), ); @@ -1392,7 +1404,6 @@ pub(crate) mod tests { NonNegativeAmount::from_u64(10000).unwrap(), &usk, &[*taddr], - &MemoBytes::empty(), NonZeroU32::new(1).unwrap() ), Ok(_) @@ -1548,6 +1559,7 @@ pub(crate) mod tests { None, OvkPolicy::Sender, NonZeroU32::new(5).unwrap(), + None ), Ok(_) ); From bcea060c8601f6a3f2523b161064154f0756af25 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 11 Oct 2023 12:08:42 -0600 Subject: [PATCH 0247/1122] Apply suggestions from code review Co-authored-by: Daira Emma Hopwood --- zcash_client_backend/src/data_api/wallet.rs | 4 +++- zcash_client_backend/src/fees.rs | 2 +- zcash_client_backend/src/fees/fixed.rs | 3 ++- zcash_client_backend/src/fees/zip317.rs | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index a6a34a6d70..31e6215747 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -118,6 +118,7 @@ where /// received note must have in the blockchain in order to be considered for being /// spent. A value of 10 confirmations is recommended and 0-conf transactions are /// not supported. +/// * `change_memo`: A memo to be included in the change output /// /// # Examples /// @@ -174,7 +175,8 @@ where /// Amount::from_u64(1).unwrap(), /// None, /// OvkPolicy::Sender, -/// 10 +/// 10, +/// None /// ) /// /// # } diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index 1698c30c02..22987c8f12 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -76,7 +76,7 @@ impl TransactionBalance { }) } - /// The change values proposed by the [`ChangeStrategy`] that computed this balance. + /// The change values proposed by the [`ChangeStrategy`] that computed this balance. pub fn proposed_change(&self) -> &[ChangeValue] { &self.proposed_change } diff --git a/zcash_client_backend/src/fees/fixed.rs b/zcash_client_backend/src/fees/fixed.rs index f4598c8249..1ab4240ff1 100644 --- a/zcash_client_backend/src/fees/fixed.rs +++ b/zcash_client_backend/src/fees/fixed.rs @@ -26,7 +26,8 @@ pub struct SingleOutputChangeStrategy { } impl SingleOutputChangeStrategy { - /// Constructs a new [`SingleOutputChangeStrategy`] with the specified fee rule. + /// Constructs a new [`SingleOutputChangeStrategy`] with the specified fee rule + /// and change memo. pub fn new(fee_rule: FixedFeeRule, change_memo: Option) -> Self { Self { fee_rule, diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index 4c1f7d6f65..5b95e08450 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -34,7 +34,7 @@ pub struct SingleOutputChangeStrategy { impl SingleOutputChangeStrategy { /// Constructs a new [`SingleOutputChangeStrategy`] with the specified ZIP 317 - /// fee parameters. + /// fee parameters and change memo. pub fn new(fee_rule: Zip317FeeRule, change_memo: Option) -> Self { Self { fee_rule, From 0de652c68720a5361490ff0cdab74a36fb8ab3d3 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 13 Oct 2023 15:28:50 -0600 Subject: [PATCH 0248/1122] zcash_client_sqlite: Fix incorrect note deduplication in `v_transactions` The `v_transactions` view is built upon the set of received notes, received note values being added to the balance for the transaction and spent notes being deducted from this balance. This fixes an error wherein if multiple identically-valued notes were spent in a transaction, only one of those notes' values was being counted as having been spent. --- zcash_client_sqlite/src/wallet/init.rs | 12 +- .../src/wallet/init/migrations.rs | 4 + .../v_transactions_note_uniqueness.rs | 160 ++++++++++++++++++ 3 files changed, 172 insertions(+), 4 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 38ab334043..49e2b79ee0 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -435,7 +435,8 @@ mod tests { "CREATE VIEW v_transactions AS WITH notes AS ( - SELECT sapling_received_notes.account AS account_id, + SELECT sapling_received_notes.id_note AS id, + sapling_received_notes.account AS account_id, transactions.block AS block, transactions.txid AS txid, 2 AS pool, @@ -457,7 +458,8 @@ mod tests { JOIN transactions ON transactions.id_tx = sapling_received_notes.tx UNION - SELECT utxos.received_by_account AS account_id, + SELECT utxos.id_utxo AS id, + utxos.received_by_account AS account_id, utxos.height AS block, utxos.prevout_txid AS txid, 0 AS pool, @@ -467,7 +469,8 @@ mod tests { 0 AS memo_present FROM utxos UNION - SELECT sapling_received_notes.account AS account_id, + SELECT sapling_received_notes.id_note AS id, + sapling_received_notes.account AS account_id, transactions.block AS block, transactions.txid AS txid, 2 AS pool, @@ -479,7 +482,8 @@ mod tests { JOIN transactions ON transactions.id_tx = sapling_received_notes.spent UNION - SELECT utxos.received_by_account AS account_id, + SELECT utxos.id_utxo AS id, + utxos.received_by_account AS account_id, transactions.block AS block, transactions.txid AS txid, 0 AS pool, diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index db8ee2194d..4c1486be50 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -12,6 +12,7 @@ mod ufvk_support; mod utxos_table; mod v_sapling_shard_unscanned_ranges; mod v_transactions_net; +mod v_transactions_note_uniqueness; mod v_transactions_shielding_balance; mod v_transactions_transparent_history; mod v_tx_outputs_use_legacy_false; @@ -48,6 +49,8 @@ pub(super) fn all_migrations( // v_sapling_shard_unscanned_ranges v_tx_outputs_use_legacy_false // | | // wallet_summaries v_transactions_shielding_balance + // | + // v_transactions_note_uniqueness vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -82,5 +85,6 @@ pub(super) fn all_migrations( Box::new(v_transactions_transparent_history::Migration), Box::new(v_tx_outputs_use_legacy_false::Migration), Box::new(v_transactions_shielding_balance::Migration), + Box::new(v_transactions_note_uniqueness::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs new file mode 100644 index 0000000000..81bf82de2f --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs @@ -0,0 +1,160 @@ +//! This migration fixes a bug in `v_transactions` where distinct but otherwise identical notes +//! were being incorrectly deduplicated. + +use std::collections::HashSet; + +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::WalletMigrationError; + +use super::v_transactions_shielding_balance; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xdba47c86_13b5_4601_94b2_0cde0abe1e45); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [v_transactions_shielding_balance::MIGRATION_ID] + .into_iter() + .collect() + } + + fn description(&self) -> &'static str { + "Fixes a bug in v_transactions that was omitting value from identically-valued notes." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + transaction.execute_batch( + "DROP VIEW v_transactions; + CREATE VIEW v_transactions AS + WITH + notes AS ( + SELECT sapling_received_notes.id_note AS id, + sapling_received_notes.account AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 2 AS pool, + sapling_received_notes.value AS value, + CASE + WHEN sapling_received_notes.is_change THEN 1 + ELSE 0 + END AS is_change, + CASE + WHEN sapling_received_notes.is_change THEN 0 + ELSE 1 + END AS received_count, + CASE + WHEN (sapling_received_notes.memo IS NULL OR sapling_received_notes.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present + FROM sapling_received_notes + JOIN transactions + ON transactions.id_tx = sapling_received_notes.tx + UNION + SELECT utxos.id_utxo AS id, + utxos.received_by_account AS account_id, + utxos.height AS block, + utxos.prevout_txid AS txid, + 0 AS pool, + utxos.value_zat AS value, + 0 AS is_change, + 1 AS received_count, + 0 AS memo_present + FROM utxos + UNION + SELECT sapling_received_notes.id_note AS id, + sapling_received_notes.account AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 2 AS pool, + -sapling_received_notes.value AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM sapling_received_notes + JOIN transactions + ON transactions.id_tx = sapling_received_notes.spent + UNION + SELECT utxos.id_utxo AS id, + utxos.received_by_account AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 0 AS pool, + -utxos.value_zat AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM utxos + JOIN transactions + ON transactions.id_tx = utxos.spent_in_tx + ), + sent_note_counts AS ( + SELECT sent_notes.from_account AS account_id, + transactions.txid AS txid, + COUNT(DISTINCT sent_notes.id_note) as sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR sapling_received_notes.tx IS NOT NULL) + THEN 0 + ELSE 1 + END + ) AS memo_count + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN sapling_received_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + WHERE COALESCE(sapling_received_notes.is_change, 0) = 0 + GROUP BY account_id, txid + ), + blocks_max_height AS ( + SELECT MAX(blocks.height) as max_height FROM blocks + ) + SELECT notes.account_id AS account_id, + notes.block AS mined_height, + notes.txid AS txid, + transactions.tx_index AS tx_index, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.is_change) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( + blocks.height IS NULL + AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height + ) AS expired_unmined + FROM notes + LEFT JOIN transactions + ON notes.txid = transactions.txid + JOIN blocks_max_height + LEFT JOIN blocks ON blocks.height = notes.block + LEFT JOIN sent_note_counts + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.txid = notes.txid + GROUP BY notes.account_id, notes.txid;" + )?; + + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + panic!("This migration cannot be reverted."); + } +} + From 11e909a2d1f8b95d225e40cbf9ce9a42a4564301 Mon Sep 17 00:00:00 2001 From: Daira Emma Hopwood Date: Wed, 18 Oct 2023 00:41:34 +0100 Subject: [PATCH 0249/1122] Add regression test for incorrect note deduplication in `v_transactions`. Signed-off-by: Daira Emma Hopwood --- .../v_transactions_note_uniqueness.rs | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs index 81bf82de2f..3636ac93f2 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs @@ -158,3 +158,93 @@ impl RusqliteMigration for Migration { } } +#[cfg(test)] +mod tests { + use rusqlite::{self, params}; + use tempfile::NamedTempFile; + + use zcash_client_backend::keys::UnifiedSpendingKey; + use zcash_primitives::{consensus::Network, zip32::AccountId}; + + use crate::{ + wallet::init::{init_wallet_db_internal, migrations::v_transactions_net}, + WalletDb, + }; + + #[test] + fn v_transactions_note_uniqueness_migration() { + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); + init_wallet_db_internal(&mut db_data, None, &[v_transactions_net::MIGRATION_ID]).unwrap(); + + // Create an account in the wallet + let usk0 = + UnifiedSpendingKey::from_seed(&db_data.params, &[0u8; 32][..], AccountId::from(0)) + .unwrap(); + let ufvk0 = usk0.to_unified_full_viewing_key(); + db_data + .conn + .execute( + "INSERT INTO accounts (account, ufvk) VALUES (0, ?)", + params![ufvk0.encode(&db_data.params)], + ) + .unwrap(); + + // Tx 0 contains two received notes, both of 2 zatoshis, that are controlled by account 0. + db_data.conn.execute_batch( + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00'); + INSERT INTO transactions (block, id_tx, txid) VALUES (0, 0, 'tx0'); + + INSERT INTO received_notes (tx, output_index, account, diversifier, value, rcm, nf, is_change) + VALUES (0, 0, 0, '', 2, '', 'nf_a', false); + INSERT INTO received_notes (tx, output_index, account, diversifier, value, rcm, nf, is_change) + VALUES (0, 3, 0, '', 2, '', 'nf_b', false);").unwrap(); + + let check_balance_delta = |db_data: &mut WalletDb, + expected_notes: i64| { + let mut q = db_data + .conn + .prepare( + "SELECT account_id, account_balance_delta, has_change, memo_count, sent_note_count, received_note_count + FROM v_transactions", + ) + .unwrap(); + let mut rows = q.query([]).unwrap(); + let mut row_count = 0; + while let Some(row) = rows.next().unwrap() { + row_count += 1; + let account: i64 = row.get(0).unwrap(); + let account_balance_delta: i64 = row.get(1).unwrap(); + let has_change: bool = row.get(2).unwrap(); + let memo_count: i64 = row.get(3).unwrap(); + let sent_note_count: i64 = row.get(4).unwrap(); + let received_note_count: i64 = row.get(5).unwrap(); + match account { + 0 => { + assert_eq!(account_balance_delta, 2 * expected_notes); + assert!(!has_change); + assert_eq!(memo_count, 0); + assert_eq!(sent_note_count, 0); + assert_eq!(received_note_count, expected_notes); + } + other => { + panic!( + "Account {:?} is not expected to exist in the wallet.", + other + ); + } + } + } + assert_eq!(row_count, 1); + }; + + // Check for the bug (#1020). + check_balance_delta(&mut db_data, 1); + + // Apply the current migration. + init_wallet_db_internal(&mut db_data, None, &[super::MIGRATION_ID]).unwrap(); + + // Now it should be correct. + check_balance_delta(&mut db_data, 2); + } +} From 2873dd4b6026f095bf76ef367152e3000deb2a30 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 13 Oct 2023 15:28:50 -0600 Subject: [PATCH 0250/1122] zcash_client_sqlite: Fix incorrect note deduplication in `v_transactions` The `v_transactions` view is built upon the set of received notes, received note values being added to the balance for the transaction and spent notes being deducted from this balance. This fixes an error wherein if multiple identically-valued notes were spent in a transaction, only one of those notes' values was being counted as having been spent. --- zcash_client_sqlite/src/wallet/init.rs | 12 +- .../src/wallet/init/migrations.rs | 4 + .../v_transactions_note_uniqueness.rs | 160 ++++++++++++++++++ 3 files changed, 172 insertions(+), 4 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 38ab334043..49e2b79ee0 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -435,7 +435,8 @@ mod tests { "CREATE VIEW v_transactions AS WITH notes AS ( - SELECT sapling_received_notes.account AS account_id, + SELECT sapling_received_notes.id_note AS id, + sapling_received_notes.account AS account_id, transactions.block AS block, transactions.txid AS txid, 2 AS pool, @@ -457,7 +458,8 @@ mod tests { JOIN transactions ON transactions.id_tx = sapling_received_notes.tx UNION - SELECT utxos.received_by_account AS account_id, + SELECT utxos.id_utxo AS id, + utxos.received_by_account AS account_id, utxos.height AS block, utxos.prevout_txid AS txid, 0 AS pool, @@ -467,7 +469,8 @@ mod tests { 0 AS memo_present FROM utxos UNION - SELECT sapling_received_notes.account AS account_id, + SELECT sapling_received_notes.id_note AS id, + sapling_received_notes.account AS account_id, transactions.block AS block, transactions.txid AS txid, 2 AS pool, @@ -479,7 +482,8 @@ mod tests { JOIN transactions ON transactions.id_tx = sapling_received_notes.spent UNION - SELECT utxos.received_by_account AS account_id, + SELECT utxos.id_utxo AS id, + utxos.received_by_account AS account_id, transactions.block AS block, transactions.txid AS txid, 0 AS pool, diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index db8ee2194d..4c1486be50 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -12,6 +12,7 @@ mod ufvk_support; mod utxos_table; mod v_sapling_shard_unscanned_ranges; mod v_transactions_net; +mod v_transactions_note_uniqueness; mod v_transactions_shielding_balance; mod v_transactions_transparent_history; mod v_tx_outputs_use_legacy_false; @@ -48,6 +49,8 @@ pub(super) fn all_migrations( // v_sapling_shard_unscanned_ranges v_tx_outputs_use_legacy_false // | | // wallet_summaries v_transactions_shielding_balance + // | + // v_transactions_note_uniqueness vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -82,5 +85,6 @@ pub(super) fn all_migrations( Box::new(v_transactions_transparent_history::Migration), Box::new(v_tx_outputs_use_legacy_false::Migration), Box::new(v_transactions_shielding_balance::Migration), + Box::new(v_transactions_note_uniqueness::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs new file mode 100644 index 0000000000..81bf82de2f --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs @@ -0,0 +1,160 @@ +//! This migration fixes a bug in `v_transactions` where distinct but otherwise identical notes +//! were being incorrectly deduplicated. + +use std::collections::HashSet; + +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::WalletMigrationError; + +use super::v_transactions_shielding_balance; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xdba47c86_13b5_4601_94b2_0cde0abe1e45); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [v_transactions_shielding_balance::MIGRATION_ID] + .into_iter() + .collect() + } + + fn description(&self) -> &'static str { + "Fixes a bug in v_transactions that was omitting value from identically-valued notes." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + transaction.execute_batch( + "DROP VIEW v_transactions; + CREATE VIEW v_transactions AS + WITH + notes AS ( + SELECT sapling_received_notes.id_note AS id, + sapling_received_notes.account AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 2 AS pool, + sapling_received_notes.value AS value, + CASE + WHEN sapling_received_notes.is_change THEN 1 + ELSE 0 + END AS is_change, + CASE + WHEN sapling_received_notes.is_change THEN 0 + ELSE 1 + END AS received_count, + CASE + WHEN (sapling_received_notes.memo IS NULL OR sapling_received_notes.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present + FROM sapling_received_notes + JOIN transactions + ON transactions.id_tx = sapling_received_notes.tx + UNION + SELECT utxos.id_utxo AS id, + utxos.received_by_account AS account_id, + utxos.height AS block, + utxos.prevout_txid AS txid, + 0 AS pool, + utxos.value_zat AS value, + 0 AS is_change, + 1 AS received_count, + 0 AS memo_present + FROM utxos + UNION + SELECT sapling_received_notes.id_note AS id, + sapling_received_notes.account AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 2 AS pool, + -sapling_received_notes.value AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM sapling_received_notes + JOIN transactions + ON transactions.id_tx = sapling_received_notes.spent + UNION + SELECT utxos.id_utxo AS id, + utxos.received_by_account AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 0 AS pool, + -utxos.value_zat AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM utxos + JOIN transactions + ON transactions.id_tx = utxos.spent_in_tx + ), + sent_note_counts AS ( + SELECT sent_notes.from_account AS account_id, + transactions.txid AS txid, + COUNT(DISTINCT sent_notes.id_note) as sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR sapling_received_notes.tx IS NOT NULL) + THEN 0 + ELSE 1 + END + ) AS memo_count + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN sapling_received_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + WHERE COALESCE(sapling_received_notes.is_change, 0) = 0 + GROUP BY account_id, txid + ), + blocks_max_height AS ( + SELECT MAX(blocks.height) as max_height FROM blocks + ) + SELECT notes.account_id AS account_id, + notes.block AS mined_height, + notes.txid AS txid, + transactions.tx_index AS tx_index, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.is_change) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( + blocks.height IS NULL + AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height + ) AS expired_unmined + FROM notes + LEFT JOIN transactions + ON notes.txid = transactions.txid + JOIN blocks_max_height + LEFT JOIN blocks ON blocks.height = notes.block + LEFT JOIN sent_note_counts + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.txid = notes.txid + GROUP BY notes.account_id, notes.txid;" + )?; + + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), Self::Error> { + panic!("This migration cannot be reverted."); + } +} + From 914c0ed5c986ec515d0843358d17dd72795c3747 Mon Sep 17 00:00:00 2001 From: Daira Emma Hopwood Date: Wed, 18 Oct 2023 00:41:34 +0100 Subject: [PATCH 0251/1122] Add regression test for incorrect note deduplication in `v_transactions`. Signed-off-by: Daira Emma Hopwood --- .../v_transactions_note_uniqueness.rs | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs index 81bf82de2f..3636ac93f2 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_note_uniqueness.rs @@ -158,3 +158,93 @@ impl RusqliteMigration for Migration { } } +#[cfg(test)] +mod tests { + use rusqlite::{self, params}; + use tempfile::NamedTempFile; + + use zcash_client_backend::keys::UnifiedSpendingKey; + use zcash_primitives::{consensus::Network, zip32::AccountId}; + + use crate::{ + wallet::init::{init_wallet_db_internal, migrations::v_transactions_net}, + WalletDb, + }; + + #[test] + fn v_transactions_note_uniqueness_migration() { + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); + init_wallet_db_internal(&mut db_data, None, &[v_transactions_net::MIGRATION_ID]).unwrap(); + + // Create an account in the wallet + let usk0 = + UnifiedSpendingKey::from_seed(&db_data.params, &[0u8; 32][..], AccountId::from(0)) + .unwrap(); + let ufvk0 = usk0.to_unified_full_viewing_key(); + db_data + .conn + .execute( + "INSERT INTO accounts (account, ufvk) VALUES (0, ?)", + params![ufvk0.encode(&db_data.params)], + ) + .unwrap(); + + // Tx 0 contains two received notes, both of 2 zatoshis, that are controlled by account 0. + db_data.conn.execute_batch( + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00'); + INSERT INTO transactions (block, id_tx, txid) VALUES (0, 0, 'tx0'); + + INSERT INTO received_notes (tx, output_index, account, diversifier, value, rcm, nf, is_change) + VALUES (0, 0, 0, '', 2, '', 'nf_a', false); + INSERT INTO received_notes (tx, output_index, account, diversifier, value, rcm, nf, is_change) + VALUES (0, 3, 0, '', 2, '', 'nf_b', false);").unwrap(); + + let check_balance_delta = |db_data: &mut WalletDb, + expected_notes: i64| { + let mut q = db_data + .conn + .prepare( + "SELECT account_id, account_balance_delta, has_change, memo_count, sent_note_count, received_note_count + FROM v_transactions", + ) + .unwrap(); + let mut rows = q.query([]).unwrap(); + let mut row_count = 0; + while let Some(row) = rows.next().unwrap() { + row_count += 1; + let account: i64 = row.get(0).unwrap(); + let account_balance_delta: i64 = row.get(1).unwrap(); + let has_change: bool = row.get(2).unwrap(); + let memo_count: i64 = row.get(3).unwrap(); + let sent_note_count: i64 = row.get(4).unwrap(); + let received_note_count: i64 = row.get(5).unwrap(); + match account { + 0 => { + assert_eq!(account_balance_delta, 2 * expected_notes); + assert!(!has_change); + assert_eq!(memo_count, 0); + assert_eq!(sent_note_count, 0); + assert_eq!(received_note_count, expected_notes); + } + other => { + panic!( + "Account {:?} is not expected to exist in the wallet.", + other + ); + } + } + } + assert_eq!(row_count, 1); + }; + + // Check for the bug (#1020). + check_balance_delta(&mut db_data, 1); + + // Apply the current migration. + init_wallet_db_internal(&mut db_data, None, &[super::MIGRATION_ID]).unwrap(); + + // Now it should be correct. + check_balance_delta(&mut db_data, 2); + } +} From b88adb4e82504ce82e8f955f8cf07514699653a1 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Wed, 18 Oct 2023 14:58:03 -0600 Subject: [PATCH 0252/1122] Release 0.8.1 --- zcash_client_sqlite/CHANGELOG.md | 5 +++++ zcash_client_sqlite/Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 2a044fc2b2..d2a2968400 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -7,6 +7,11 @@ and this library adheres to Rust's notion of ## [Unreleased] +## [0.8.1] - 2023-10-18 + +### Fixed +- Fixed a bug in `v_transactions` that was omitting value from identically-valued notes + ## [0.8.0] - 2023-09-25 ### Notable Changes diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index bb2717717f..b4605a08e3 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_sqlite" description = "An SQLite-based Zcash light client" -version = "0.8.0" +version = "0.8.1" authors = [ "Jack Grigg ", "Kris Nuttycombe " From 289f1fdf302e3e842a62903213c495c87a441913 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Wed, 18 Oct 2023 14:58:03 -0600 Subject: [PATCH 0253/1122] Release 0.8.1 Signed-off-by: Daira Emma Hopwood --- Cargo.lock | 2 +- zcash_client_sqlite/CHANGELOG.md | 5 +++++ zcash_client_sqlite/Cargo.toml | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e594db32ce..57991fdd60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2988,7 +2988,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" -version = "0.8.0" +version = "0.8.1" dependencies = [ "assert_matches", "bs58", diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 2a044fc2b2..405492a64d 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -5,6 +5,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.8.1] - 2023-10-18 + +### Fixed +- Fixed a bug in `v_transactions` that was omitting value from identically-valued notes + ## [Unreleased] ## [0.8.0] - 2023-09-25 diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index bb2717717f..b4605a08e3 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_sqlite" description = "An SQLite-based Zcash light client" -version = "0.8.0" +version = "0.8.1" authors = [ "Jack Grigg ", "Kris Nuttycombe " From f7527e14c9763a6786055e38dca37b32ad4bcaaa Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 9 Oct 2023 18:05:48 -0600 Subject: [PATCH 0254/1122] Use `NonNegativeAmount` for note and utxo value fields --- zcash_client_backend/CHANGELOG.md | 16 ++- zcash_client_backend/src/data_api.rs | 6 +- zcash_client_backend/src/data_api/error.rs | 15 ++- zcash_client_backend/src/data_api/wallet.rs | 6 +- .../src/data_api/wallet/input_selection.rs | 37 +++--- zcash_client_backend/src/fees.rs | 16 +-- zcash_client_backend/src/fees/fixed.rs | 109 ++++++++-------- zcash_client_backend/src/fees/zip317.rs | 121 +++++++++--------- zcash_client_backend/src/wallet.rs | 12 +- zcash_client_backend/src/zip321.rs | 49 ++++--- zcash_client_sqlite/src/chain.rs | 9 +- zcash_client_sqlite/src/lib.rs | 7 +- zcash_client_sqlite/src/wallet.rs | 23 ++-- .../init/migrations/add_transaction_views.rs | 4 +- zcash_client_sqlite/src/wallet/sapling.rs | 39 +++--- zcash_extensions/src/transparent/demo.rs | 12 +- zcash_primitives/CHANGELOG.md | 17 +++ zcash_primitives/src/transaction/builder.rs | 70 ++-------- .../src/transaction/components/amount.rs | 19 ++- .../transaction/components/sapling/builder.rs | 10 +- .../transaction/components/sapling/fees.rs | 6 +- .../src/transaction/components/transparent.rs | 8 +- .../components/transparent/builder.rs | 28 ++-- .../components/transparent/fees.rs | 6 +- .../src/transaction/components/tze.rs | 2 +- zcash_primitives/src/transaction/sighash.rs | 5 +- zcash_primitives/src/transaction/tests.rs | 15 ++- 27 files changed, 339 insertions(+), 328 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 9b81117e55..9c1da9adac 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -27,8 +27,12 @@ and this library adheres to Rust's notion of - The `NoteMismatch` variant of `data_api::error::Error` now wraps a `data_api::NoteId` instead of a backend-specific note identifier. The related `NoteRef` type parameter has been removed from `data_api::error::Error`. - - `wallet::create_spend_to_address` now takes a `NonNegativeAmount` rather than - an `Amount`. + - `SentTransactionOutput::value` is now represented as `NonNegativeAmount` instead of + `Amount`, and constructors and accessors have been updated accordingly. + - The `available` and `required` fields of `data_api::error::Error::InsufficientFunds` + are now represented as `NonNegativeAmount` instead of `Amount`. + - `data_api::wallet::create_spend_to_address` now takes its `amount` argument as + as `NonNegativeAmount` instead of `Amount`. - All uses of `Amount` in `data_api::wallet::input_selection` have been replaced with `NonNegativeAmount`. - `wallet::shield_transparent_funds` no longer @@ -50,6 +54,14 @@ and this library adheres to Rust's notion of accept an additional `change_memo` argument. - All uses of `Amount` in `zcash_client_backend::fees` have been replaced with `NonNegativeAmount`. +- `zcash_client_backend::wallet::WalletTransparentOutput::value` is now represented + as `NonNegativeAmount` instead of `Amount`, and constructors and accessors have been + updated accordingly. +- `zcash_client_backend::wallet::ReceivedSaplingNote::value` is now represented + as `NonNegativeAmount` instead of `Amount`, and constructors and accessors have been + updated accordingly. +- Almost all uses of `Amount` in `zcash_client_backend::zip321` have been replaced + with `NonNegativeAmount`. ## [0.10.0] - 2023-09-25 diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index c710da193f..1253dbf98e 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -630,7 +630,7 @@ pub enum Recipient { pub struct SentTransactionOutput { output_index: usize, recipient: Recipient, - value: Amount, + value: NonNegativeAmount, memo: Option, sapling_change_to: Option<(AccountId, sapling::Note)>, } @@ -639,7 +639,7 @@ impl SentTransactionOutput { pub fn from_parts( output_index: usize, recipient: Recipient, - value: Amount, + value: NonNegativeAmount, memo: Option, sapling_change_to: Option<(AccountId, sapling::Note)>, ) -> Self { @@ -667,7 +667,7 @@ impl SentTransactionOutput { &self.recipient } /// Returns the value of the newly created output. - pub fn value(&self) -> Amount { + pub fn value(&self) -> NonNegativeAmount { self.value } /// Returns the memo that was attached to the output, if any. This will only be `None` diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index e2b1c5bcfd..0fcde032fa 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -4,13 +4,11 @@ use std::error; use std::fmt::{self, Debug, Display}; use shardtree::error::ShardTreeError; +use zcash_primitives::transaction::components::amount::NonNegativeAmount; use zcash_primitives::{ transaction::{ builder, - components::{ - amount::{Amount, BalanceError}, - sapling, transparent, - }, + components::{amount::BalanceError, sapling, transparent}, }, zip32::AccountId, }; @@ -45,7 +43,10 @@ pub enum Error { BalanceError(BalanceError), /// Unable to create a new spend because the wallet balance is not sufficient. - InsufficientFunds { available: Amount, required: Amount }, + InsufficientFunds { + available: NonNegativeAmount, + required: NonNegativeAmount, + }, /// The wallet must first perform a scan of the blockchain before other /// operations can be performed. @@ -110,8 +111,8 @@ where Error::InsufficientFunds { available, required } => write!( f, "Insufficient balance (have {}, need {} including fee)", - i64::from(*available), - i64::from(*required) + u64::from(*available), + u64::from(*required) ), Error::ScanRequired => write!(f, "Must scan blocks first"), Error::Builder(e) => write!(f, "An error occurred building the transaction: {}", e), diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 43d72e7488..98454d3555 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -215,7 +215,7 @@ where { let req = zip321::TransactionRequest::new(vec![Payment { recipient_address: to.clone(), - amount: amount.into(), + amount, memo, label: None, message: None, @@ -591,7 +591,7 @@ where builder.add_sapling_output( internal_ovk(), dfvk.change_address().1, - value.into(), + *value, memo.clone(), )?; sapling_output_meta.push(( @@ -599,7 +599,7 @@ where account, PoolType::Shielded(ShieldedProtocol::Sapling), ), - value.into(), + *value, Some(memo), )) } diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 4bdbede17b..1c3685ed03 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -10,7 +10,7 @@ use zcash_primitives::{ legacy::TransparentAddress, transaction::{ components::{ - amount::{Amount, BalanceError, NonNegativeAmount}, + amount::{BalanceError, NonNegativeAmount}, sapling::fees as sapling, OutPoint, TxOut, }, @@ -35,7 +35,10 @@ pub enum InputSelectorError { Selection(SelectorErrT), /// Insufficient funds were available to satisfy the payment request that inputs were being /// selected to attempt to satisfy. - InsufficientFunds { available: Amount, required: Amount }, + InsufficientFunds { + available: NonNegativeAmount, + required: NonNegativeAmount, + }, /// The data source does not have enough information to choose an expiry height /// for the transaction. SyncRequired, @@ -60,8 +63,8 @@ impl fmt::Display for InputSelectorError write!( f, "Insufficient balance (have {}, need {} including fee)", - i64::from(*available), - i64::from(*required) + u64::from(*available), + u64::from(*required) ), InputSelectorError::SyncRequired => { write!(f, "Insufficient chain data is available, sync required.") @@ -270,17 +273,17 @@ impl From } } -pub(crate) struct SaplingPayment(Amount); +pub(crate) struct SaplingPayment(NonNegativeAmount); #[cfg(test)] impl SaplingPayment { - pub(crate) fn new(amount: Amount) -> Self { + pub(crate) fn new(amount: NonNegativeAmount) -> Self { SaplingPayment(amount) } } impl sapling::OutputView for SaplingPayment { - fn value(&self) -> Amount { + fn value(&self) -> NonNegativeAmount { self.0 } } @@ -338,10 +341,7 @@ where let mut transparent_outputs = vec![]; let mut sapling_outputs = vec![]; - let mut output_total = Amount::zero(); for payment in transaction_request.payments() { - output_total = (output_total + payment.amount).ok_or(BalanceError::Overflow)?; - let mut push_transparent = |taddr: TransparentAddress| { transparent_outputs.push(TxOut { value: payment.amount, @@ -374,8 +374,8 @@ where } let mut sapling_inputs: Vec> = vec![]; - let mut prior_available = Amount::zero(); - let mut amount_required = Amount::zero(); + let mut prior_available = NonNegativeAmount::ZERO; + let mut amount_required = NonNegativeAmount::ZERO; let mut exclude: Vec = vec![]; // This loop is guaranteed to terminate because on each iteration we check that the amount // of funds selected is strictly increasing. The loop will either return a successful @@ -414,13 +414,18 @@ where } sapling_inputs = wallet_db - .select_spendable_sapling_notes(account, amount_required, anchor_height, &exclude) + .select_spendable_sapling_notes( + account, + amount_required.into(), + anchor_height, + &exclude, + ) .map_err(InputSelectorError::DataSource)?; let new_available = sapling_inputs .iter() .map(|n| n.value()) - .sum::>() + .sum::>() .ok_or(BalanceError::Overflow)?; if new_available <= prior_available { @@ -506,8 +511,8 @@ where }) } else { Err(InputSelectorError::InsufficientFunds { - available: balance.total().into(), - required: shielding_threshold.into(), + available: balance.total(), + required: shielding_threshold, }) } } diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index 22987c8f12..5ddcffc1bf 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -5,7 +5,7 @@ use zcash_primitives::{ memo::MemoBytes, transaction::{ components::{ - amount::{Amount, BalanceError, NonNegativeAmount}, + amount::{BalanceError, NonNegativeAmount}, sapling::fees as sapling, transparent::fees as transparent, OutPoint, @@ -100,10 +100,10 @@ pub enum ChangeError { /// required outputs and fees. InsufficientFunds { /// The total of the inputs provided to change selection - available: Amount, + available: NonNegativeAmount, /// The total amount of input value required to fund the requested outputs, /// including the required fees. - required: Amount, + required: NonNegativeAmount, }, /// Some of the inputs provided to the transaction were determined to currently have no /// economic value (i.e. their inclusion in a transaction causes fees to rise in an amount @@ -127,8 +127,8 @@ impl fmt::Display for ChangeError { } => write!( f, "Insufficient funds: required {} zatoshis, but only {} zatoshis were available.", - i64::from(required), - i64::from(available) + u64::from(*required), + u64::from(*available) ), ChangeError::DustInputs { transparent, @@ -235,7 +235,7 @@ pub trait ChangeStrategy { #[cfg(test)] pub(crate) mod tests { use zcash_primitives::transaction::components::{ - amount::Amount, + amount::NonNegativeAmount, sapling::fees as sapling, transparent::{fees as transparent, OutPoint, TxOut}, }; @@ -257,14 +257,14 @@ pub(crate) mod tests { pub(crate) struct TestSaplingInput { pub note_id: u32, - pub value: Amount, + pub value: NonNegativeAmount, } impl sapling::InputView for TestSaplingInput { fn note_id(&self) -> &u32 { &self.note_id } - fn value(&self) -> Amount { + fn value(&self) -> NonNegativeAmount { self.value } } diff --git a/zcash_client_backend/src/fees/fixed.rs b/zcash_client_backend/src/fees/fixed.rs index 1ab4240ff1..773f1d8df4 100644 --- a/zcash_client_backend/src/fees/fixed.rs +++ b/zcash_client_backend/src/fees/fixed.rs @@ -1,12 +1,11 @@ //! Change strategies designed for use with a fixed fee. -use std::cmp::Ordering; use zcash_primitives::{ consensus::{self, BlockHeight}, memo::MemoBytes, transaction::{ components::{ - amount::{Amount, BalanceError, NonNegativeAmount}, + amount::{BalanceError, NonNegativeAmount}, sapling::fees as sapling, transparent::fees as transparent, }, @@ -89,9 +88,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { ) .unwrap(); // fixed::FeeRule::fee_required is infallible. - let total_in = (t_in + sapling_in) - .and_then(|v| NonNegativeAmount::try_from(v).ok()) - .ok_or(BalanceError::Overflow)?; + let total_in = (t_in + sapling_in).ok_or(BalanceError::Overflow)?; if (!transparent_inputs.is_empty() || !sapling_inputs.is_empty()) && fee_amount > total_in { // For the fixed-fee selection rule, the only time we consider inputs dust is when the fee @@ -109,62 +106,57 @@ impl ChangeStrategy for SingleOutputChangeStrategy { .collect(), }) } else { - let total_out = [t_out, sapling_out, fee_amount.into()] + let total_out = [t_out, sapling_out, fee_amount] .iter() - .sum::>() + .sum::>() .ok_or(BalanceError::Overflow)?; let overflow = |_| ChangeError::StrategyError(BalanceError::Overflow); - let proposed_change = - (Amount::from(total_in) - total_out).ok_or(BalanceError::Underflow)?; - match proposed_change.cmp(&Amount::zero()) { - Ordering::Less => Err(ChangeError::InsufficientFunds { - available: total_in.into(), - required: total_out, - }), - Ordering::Equal => TransactionBalance::new(vec![], fee_amount).map_err(overflow), - Ordering::Greater => { - let proposed_change = NonNegativeAmount::try_from(proposed_change).unwrap(); - let dust_threshold = dust_output_policy - .dust_threshold() - .unwrap_or_else(|| self.fee_rule.fixed_fee()); - if dust_threshold > proposed_change { - match dust_output_policy.action() { - DustAction::Reject => { - let shortfall = (dust_threshold - proposed_change) - .ok_or(BalanceError::Underflow)?; - Err(ChangeError::InsufficientFunds { - available: total_in.into(), - required: (total_in + shortfall) - .ok_or(BalanceError::Overflow)? - .into(), - }) - } - DustAction::AllowDustChange => TransactionBalance::new( - vec![ChangeValue::sapling( - proposed_change, - self.change_memo.clone(), - )], - fee_amount, - ) - .map_err(overflow), - DustAction::AddDustToFee => TransactionBalance::new( - vec![], - (fee_amount + proposed_change).ok_or(BalanceError::Overflow)?, - ) - .map_err(overflow), + let proposed_change = (total_in - total_out).ok_or(ChangeError::InsufficientFunds { + available: total_in, + required: total_out, + })?; + if proposed_change == NonNegativeAmount::ZERO { + TransactionBalance::new(vec![], fee_amount).map_err(overflow) + } else { + let dust_threshold = dust_output_policy + .dust_threshold() + .unwrap_or_else(|| self.fee_rule.fixed_fee()); + + if dust_threshold > proposed_change { + match dust_output_policy.action() { + DustAction::Reject => { + let shortfall = (dust_threshold - proposed_change) + .ok_or(BalanceError::Underflow)?; + Err(ChangeError::InsufficientFunds { + available: total_in, + required: (total_in + shortfall).ok_or(BalanceError::Overflow)?, + }) } - } else { - TransactionBalance::new( + DustAction::AllowDustChange => TransactionBalance::new( vec![ChangeValue::sapling( proposed_change, self.change_memo.clone(), )], fee_amount, ) - .map_err(overflow) + .map_err(overflow), + DustAction::AddDustToFee => TransactionBalance::new( + vec![], + (fee_amount + proposed_change).ok_or(BalanceError::Overflow)?, + ) + .map_err(overflow), } + } else { + TransactionBalance::new( + vec![ChangeValue::sapling( + proposed_change, + self.change_memo.clone(), + )], + fee_amount, + ) + .map_err(overflow) } } } @@ -176,10 +168,7 @@ mod tests { use zcash_primitives::{ consensus::{Network, NetworkUpgrade, Parameters}, transaction::{ - components::{ - amount::{Amount, NonNegativeAmount}, - transparent::TxOut, - }, + components::{amount::NonNegativeAmount, transparent::TxOut}, fees::fixed::FeeRule as FixedFeeRule, }, }; @@ -209,9 +198,11 @@ mod tests { &Vec::::new(), &[TestSaplingInput { note_id: 0, - value: Amount::from_u64(60000).unwrap(), + value: NonNegativeAmount::const_from_u64(60000), }], - &[SaplingPayment::new(Amount::from_u64(40000).unwrap())], + &[SaplingPayment::new(NonNegativeAmount::const_from_u64( + 40000, + ))], &DustOutputPolicy::default(), ); @@ -240,22 +231,24 @@ mod tests { &[ TestSaplingInput { note_id: 0, - value: Amount::from_u64(40000).unwrap(), + value: NonNegativeAmount::const_from_u64(40000), }, // enough to pay a fee, plus dust TestSaplingInput { note_id: 0, - value: Amount::from_u64(10100).unwrap(), + value: NonNegativeAmount::const_from_u64(10100), }, ], - &[SaplingPayment::new(Amount::from_u64(40000).unwrap())], + &[SaplingPayment::new(NonNegativeAmount::const_from_u64( + 40000, + ))], &DustOutputPolicy::default(), ); assert_matches!( result, Err(ChangeError::InsufficientFunds { available, required }) - if available == Amount::from_u64(50100).unwrap() && required == Amount::from_u64(60000).unwrap() + if available == NonNegativeAmount::const_from_u64(50100) && required == NonNegativeAmount::const_from_u64(60000) ); } } diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index 5b95e08450..4063041ce7 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -3,14 +3,13 @@ //! Change selection in ZIP 317 requires careful handling of low-valued inputs //! to ensure that inputs added to a transaction do not cause fees to rise by //! an amount greater than their value. -use core::cmp::Ordering; use zcash_primitives::{ consensus::{self, BlockHeight}, memo::MemoBytes, transaction::{ components::{ - amount::{Amount, BalanceError, NonNegativeAmount}, + amount::{BalanceError, NonNegativeAmount}, sapling::fees as sapling, transparent::fees as transparent, }, @@ -66,7 +65,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { .filter_map(|i| { // for now, we're just assuming p2pkh inputs, so we don't check the size of the input // script - if i.coin().value < self.fee_rule.marginal_fee().into() { + if i.coin().value < self.fee_rule.marginal_fee() { Some(i.outpoint().clone()) } else { None @@ -77,7 +76,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { let mut sapling_dust: Vec<_> = sapling_inputs .iter() .filter_map(|i| { - if i.value() < self.fee_rule.marginal_fee().into() { + if i.value() < self.fee_rule.marginal_fee() { Some(i.note_id().clone()) } else { None @@ -179,59 +178,56 @@ impl ChangeStrategy for SingleOutputChangeStrategy { let total_in = (t_in + sapling_in).ok_or_else(overflow)?; - let total_out = [t_out, sapling_out, fee_amount.into()] + let total_out = [t_out, sapling_out, fee_amount] .iter() - .sum::>() + .sum::>() .ok_or_else(overflow)?; - let proposed_change = (total_in - total_out).ok_or_else(underflow)?; - match proposed_change.cmp(&Amount::zero()) { - Ordering::Less => Err(ChangeError::InsufficientFunds { - available: total_in, - required: total_out, - }), - Ordering::Equal => TransactionBalance::new(vec![], fee_amount).map_err(|_| overflow()), - Ordering::Greater => { - let proposed_change = NonNegativeAmount::try_from(proposed_change).unwrap(); - let dust_threshold = dust_output_policy - .dust_threshold() - .unwrap_or_else(|| self.fee_rule.marginal_fee()); - - if dust_threshold > proposed_change { - match dust_output_policy.action() { - DustAction::Reject => { - let shortfall = - (dust_threshold - proposed_change).ok_or_else(underflow)?; - - Err(ChangeError::InsufficientFunds { - available: total_in, - required: (total_in + shortfall.into()).ok_or_else(overflow)?, - }) - } - DustAction::AllowDustChange => TransactionBalance::new( - vec![ChangeValue::sapling( - proposed_change, - self.change_memo.clone(), - )], - fee_amount, - ) - .map_err(|_| overflow()), - DustAction::AddDustToFee => TransactionBalance::new( - vec![], - (fee_amount + proposed_change).ok_or_else(overflow)?, - ) - .map_err(|_| overflow()), + let proposed_change = (total_in - total_out).ok_or(ChangeError::InsufficientFunds { + available: total_in, + required: total_out, + })?; + + if proposed_change == NonNegativeAmount::ZERO { + TransactionBalance::new(vec![], fee_amount).map_err(|_| overflow()) + } else { + let dust_threshold = dust_output_policy + .dust_threshold() + .unwrap_or_else(|| self.fee_rule.marginal_fee()); + + if dust_threshold > proposed_change { + match dust_output_policy.action() { + DustAction::Reject => { + let shortfall = (dust_threshold - proposed_change).ok_or_else(underflow)?; + + Err(ChangeError::InsufficientFunds { + available: total_in, + required: (total_in + shortfall).ok_or_else(overflow)?, + }) } - } else { - TransactionBalance::new( + DustAction::AllowDustChange => TransactionBalance::new( vec![ChangeValue::sapling( proposed_change, self.change_memo.clone(), )], fee_amount, ) - .map_err(|_| overflow()) + .map_err(|_| overflow()), + DustAction::AddDustToFee => TransactionBalance::new( + vec![], + (fee_amount + proposed_change).ok_or_else(overflow)?, + ) + .map_err(|_| overflow()), } + } else { + TransactionBalance::new( + vec![ChangeValue::sapling( + proposed_change, + self.change_memo.clone(), + )], + fee_amount, + ) + .map_err(|_| overflow()) } } } @@ -244,10 +240,7 @@ mod tests { consensus::{Network, NetworkUpgrade, Parameters}, legacy::Script, transaction::{ - components::{ - amount::{Amount, NonNegativeAmount}, - transparent::TxOut, - }, + components::{amount::NonNegativeAmount, transparent::TxOut}, fees::zip317::FeeRule as Zip317FeeRule, }, }; @@ -275,9 +268,11 @@ mod tests { &Vec::::new(), &[TestSaplingInput { note_id: 0, - value: Amount::from_u64(55000).unwrap(), + value: NonNegativeAmount::const_from_u64(55000), }], - &[SaplingPayment::new(Amount::from_u64(40000).unwrap())], + &[SaplingPayment::new(NonNegativeAmount::const_from_u64( + 40000, + ))], &DustOutputPolicy::default(), ); @@ -301,12 +296,12 @@ mod tests { .unwrap(), &Vec::::new(), &[TxOut { - value: Amount::from_u64(40000).unwrap(), + value: NonNegativeAmount::const_from_u64(40000), script_pubkey: Script(vec![]), }], &[TestSaplingInput { note_id: 0, - value: Amount::from_u64(55000).unwrap(), + value: NonNegativeAmount::const_from_u64(55000), }], &Vec::::new(), &DustOutputPolicy::default(), @@ -334,14 +329,16 @@ mod tests { &[ TestSaplingInput { note_id: 0, - value: Amount::from_u64(49000).unwrap(), + value: NonNegativeAmount::const_from_u64(49000), }, TestSaplingInput { note_id: 1, - value: Amount::from_u64(1000).unwrap(), + value: NonNegativeAmount::const_from_u64(1000), }, ], - &[SaplingPayment::new(Amount::from_u64(40000).unwrap())], + &[SaplingPayment::new(NonNegativeAmount::const_from_u64( + 40000, + ))], &DustOutputPolicy::default(), ); @@ -367,18 +364,20 @@ mod tests { &[ TestSaplingInput { note_id: 0, - value: Amount::from_u64(29000).unwrap(), + value: NonNegativeAmount::const_from_u64(29000), }, TestSaplingInput { note_id: 1, - value: Amount::from_u64(20000).unwrap(), + value: NonNegativeAmount::const_from_u64(20000), }, TestSaplingInput { note_id: 2, - value: Amount::from_u64(1000).unwrap(), + value: NonNegativeAmount::const_from_u64(1000), }, ], - &[SaplingPayment::new(Amount::from_u64(40000).unwrap())], + &[SaplingPayment::new(NonNegativeAmount::const_from_u64( + 40000, + ))], &DustOutputPolicy::default(), ); diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index c7467c8a69..63863f2e07 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -10,9 +10,9 @@ use zcash_primitives::{ sapling, transaction::{ components::{ + amount::NonNegativeAmount, sapling::fees as sapling_fees, transparent::{self, OutPoint, TxOut}, - Amount, }, TxId, }, @@ -69,7 +69,7 @@ impl WalletTransparentOutput { &self.recipient_address } - pub fn value(&self) -> Amount { + pub fn value(&self) -> NonNegativeAmount { self.txout.value } } @@ -181,7 +181,7 @@ pub struct ReceivedSaplingNote { txid: TxId, output_index: u16, diversifier: sapling::Diversifier, - note_value: Amount, + note_value: NonNegativeAmount, rseed: sapling::Rseed, note_commitment_tree_position: Position, } @@ -192,7 +192,7 @@ impl ReceivedSaplingNote { txid: TxId, output_index: u16, diversifier: sapling::Diversifier, - note_value: Amount, + note_value: NonNegativeAmount, rseed: sapling::Rseed, note_commitment_tree_position: Position, ) -> Self { @@ -220,7 +220,7 @@ impl ReceivedSaplingNote { pub fn diversifier(&self) -> sapling::Diversifier { self.diversifier } - pub fn value(&self) -> Amount { + pub fn value(&self) -> NonNegativeAmount { self.note_value } pub fn rseed(&self) -> sapling::Rseed { @@ -236,7 +236,7 @@ impl sapling_fees::InputView for ReceivedSaplingNote &self.note_id } - fn value(&self) -> Amount { + fn value(&self) -> NonNegativeAmount { self.note_value } } diff --git a/zcash_client_backend/src/zip321.rs b/zcash_client_backend/src/zip321.rs index a26c3cdf48..28523f677a 100644 --- a/zcash_client_backend/src/zip321.rs +++ b/zcash_client_backend/src/zip321.rs @@ -15,7 +15,7 @@ use nom::{ use zcash_primitives::{ consensus, memo::{self, MemoBytes}, - transaction::components::Amount, + transaction::components::amount::NonNegativeAmount, }; #[cfg(any(test, feature = "test-dependencies"))] @@ -68,7 +68,7 @@ pub struct Payment { /// The payment address to which the payment should be sent. pub recipient_address: RecipientAddress, /// The amount of the payment that is being requested. - pub amount: Amount, + pub amount: NonNegativeAmount, /// A memo that, if included, must be provided with the payment. /// If a memo is present and [`recipient_address`] is not a shielded /// address, the wallet should report an error. @@ -298,7 +298,9 @@ mod render { use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; use zcash_primitives::{ - consensus, transaction::components::amount::COIN, transaction::components::Amount, + consensus, + transaction::components::amount::COIN, + transaction::components::{amount::NonNegativeAmount, Amount}, }; use super::{memo_to_base64, MemoBytes, RecipientAddress}; @@ -369,8 +371,8 @@ mod render { /// Constructs an "amount" key/value pair containing the encoded ZEC amount /// at the specified parameter index. - pub fn amount_param(amount: Amount, idx: Option) -> Option { - amount_str(amount).map(|s| format!("amount{}={}", param_index(idx), s)) + pub fn amount_param(amount: NonNegativeAmount, idx: Option) -> Option { + amount_str(amount.into()).map(|s| format!("amount{}={}", param_index(idx), s)) } /// Constructs a "memo" key/value pair containing the base64URI-encoded memo @@ -403,7 +405,9 @@ mod parse { }; use percent_encoding::percent_decode; use zcash_primitives::{ - consensus, transaction::components::amount::COIN, transaction::components::Amount, + consensus, + transaction::components::amount::COIN, + transaction::components::{amount::NonNegativeAmount, Amount}, }; use crate::address::RecipientAddress; @@ -415,7 +419,7 @@ mod parse { #[derive(Debug, PartialEq, Eq)] pub enum Param { Addr(Box), - Amount(Amount), + Amount(NonNegativeAmount), Memo(MemoBytes), Label(String), Message(String), @@ -462,7 +466,7 @@ mod parse { let mut payment = Payment { recipient_address: *addr.ok_or(Zip321Error::RecipientMissing(i))?, - amount: Amount::zero(), + amount: NonNegativeAmount::ZERO, memo: None, label: None, message: None, @@ -618,8 +622,12 @@ mod parse { )), "amount" => parse_amount(value) - .map(|(_, a)| Param::Amount(a)) - .map_err(|e| e.to_string()), + .map_err(|e| e.to_string()) + .and_then(|(_, a)| { + NonNegativeAmount::try_from(a) + .map_err(|_| "Payment amount must be nonnegative.".to_owned()) + }) + .map(Param::Amount), "label" => percent_decode(value.as_bytes()) .decode_utf8() @@ -753,7 +761,7 @@ mod tests { use zcash_primitives::{ consensus::{Parameters, TEST_NETWORK}, memo::Memo, - transaction::components::Amount, + transaction::components::{amount::NonNegativeAmount, Amount}, }; use crate::address::RecipientAddress; @@ -815,7 +823,7 @@ mod tests { payments: vec![ Payment { recipient_address: RecipientAddress::Shielded(decode_payment_address(TEST_NETWORK.hrp_sapling_payment_address(), "ztestsapling1n65uaftvs2g7075q2x2a04shfk066u3lldzxsrprfrqtzxnhc9ps73v4lhx4l9yfxj46sl0q90k").unwrap()), - amount: Amount::from_u64(376876902796286).unwrap(), + amount: NonNegativeAmount::const_from_u64(376876902796286), memo: None, label: None, message: Some("".to_string()), @@ -836,7 +844,7 @@ mod tests { payments: vec![ Payment { recipient_address: RecipientAddress::Shielded(decode_payment_address(TEST_NETWORK.hrp_sapling_payment_address(), "ztestsapling1n65uaftvs2g7075q2x2a04shfk066u3lldzxsrprfrqtzxnhc9ps73v4lhx4l9yfxj46sl0q90k").unwrap()), - amount: Amount::from_u64(0).unwrap(), + amount: NonNegativeAmount::ZERO, memo: None, label: None, message: None, @@ -854,7 +862,7 @@ mod tests { payments: vec![ Payment { recipient_address: RecipientAddress::Shielded(decode_payment_address(TEST_NETWORK.hrp_sapling_payment_address(), "ztestsapling1n65uaftvs2g7075q2x2a04shfk066u3lldzxsrprfrqtzxnhc9ps73v4lhx4l9yfxj46sl0q90k").unwrap()), - amount: Amount::from_u64(0).unwrap(), + amount: NonNegativeAmount::ZERO, memo: None, label: None, message: Some("".to_string()), @@ -891,7 +899,7 @@ mod tests { let v1r = TransactionRequest::from_uri(&TEST_NETWORK, valid_1).unwrap(); assert_eq!( v1r.payments.get(0).map(|p| p.amount), - Some(Amount::from_u64(100000000).unwrap()) + Some(NonNegativeAmount::const_from_u64(100000000)) ); let valid_2 = "zcash:?address=tmEZhbWHTpdKMw5it8YDspUXSMGQyFwovpU&amount=123.456&address.1=ztestsapling10yy2ex5dcqkclhc7z7yrnjq2z6feyjad56ptwlfgmy77dmaqqrl9gyhprdx59qgmsnyfska2kez&amount.1=0.789&memo.1=VGhpcyBpcyBhIHVuaWNvZGUgbWVtbyDinKjwn6aE8J-PhvCfjok"; @@ -899,11 +907,11 @@ mod tests { v2r.normalize(&TEST_NETWORK); assert_eq!( v2r.payments.get(0).map(|p| p.amount), - Some(Amount::from_u64(12345600000).unwrap()) + Some(NonNegativeAmount::const_from_u64(12345600000)) ); assert_eq!( v2r.payments.get(1).map(|p| p.amount), - Some(Amount::from_u64(78900000).unwrap()) + Some(NonNegativeAmount::const_from_u64(78900000)) ); // valid; amount just less than MAX_MONEY @@ -912,7 +920,7 @@ mod tests { let v3r = TransactionRequest::from_uri(&TEST_NETWORK, valid_3).unwrap(); assert_eq!( v3r.payments.get(0).map(|p| p.amount), - Some(Amount::from_u64(2099999999999999u64).unwrap()) + Some(NonNegativeAmount::const_from_u64(2099999999999999u64)) ); // valid; MAX_MONEY @@ -921,7 +929,7 @@ mod tests { let v4r = TransactionRequest::from_uri(&TEST_NETWORK, valid_4).unwrap(); assert_eq!( v4r.payments.get(0).map(|p| p.amount), - Some(Amount::from_u64(2100000000000000u64).unwrap()) + Some(NonNegativeAmount::const_from_u64(2100000000000000u64)) ); } @@ -1004,7 +1012,8 @@ mod tests { } #[test] - fn prop_zip321_roundtrip_amount(amt in arb_nonnegative_amount()) { + fn prop_zip321_roundtrip_amount(nn_amt in arb_nonnegative_amount()) { + let amt = Amount::from(nn_amt); let amt_str = amount_str(amt).unwrap(); assert_eq!(amt, parse_amount(&amt_str).unwrap().1); } diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 9756745658..011713ca0c 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -326,10 +326,7 @@ mod tests { use zcash_primitives::{ block::BlockHash, - transaction::{ - components::{amount::NonNegativeAmount, Amount}, - fees::zip317::FeeRule, - }, + transaction::{components::amount::NonNegativeAmount, fees::zip317::FeeRule}, zip32::ExtendedSpendingKey, }; @@ -518,13 +515,13 @@ mod tests { st.scan_cached_blocks(h2, 1); assert_eq!( st.get_total_balance(AccountId::from(0)), - NonNegativeAmount::from_u64(150_000).unwrap() + NonNegativeAmount::const_from_u64(150_000) ); // We can spend the received notes let req = TransactionRequest::new(vec![Payment { recipient_address: RecipientAddress::Shielded(dfvk.default_address().1), - amount: Amount::from_u64(110_000).unwrap(), + amount: NonNegativeAmount::const_from_u64(110_000), memo: None, label: None, message: None, diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 6025838b5e..c35a42d188 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -55,7 +55,10 @@ use zcash_primitives::{ memo::{Memo, MemoBytes}, sapling, transaction::{ - components::{amount::Amount, OutPoint}, + components::{ + amount::{Amount, NonNegativeAmount}, + OutPoint, + }, Transaction, TxId, }, zip32::{AccountId, DiversifierIndex, ExtendedFullViewingKey}, @@ -596,7 +599,7 @@ impl WalletWrite for WalletDb tx_ref, output.index, &recipient, - Amount::from_u64(output.note.value().inner()).map_err(|_| { + NonNegativeAmount::from_u64(output.note.value().inner()).map_err(|_| { SqliteClientError::CorruptedData( "Note value is not a valid Zcash amount.".to_string(), ) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index cb29419b82..a74ee67f0e 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -1345,7 +1345,10 @@ pub(crate) fn get_unspent_transparent_outputs( let index: u32 = row.get(1)?; let script_pubkey = Script(row.get(2)?); - let value = Amount::from_i64(row.get(3)?).unwrap(); + let value_raw: i64 = row.get(3)?; + let value = NonNegativeAmount::from_nonnegative_i64(value_raw).map_err(|_| { + SqliteClientError::CorruptedData(format!("Negative utxo value: {}", value_raw)) + })?; let height: u32 = row.get(4)?; let outpoint = OutPoint::new(txid_bytes, index); @@ -1642,7 +1645,7 @@ pub(crate) fn put_legacy_transparent_utxo( ":received_by_account": &u32::from(received_by_account), ":address": &output.recipient_address().encode(params), ":script": &output.txout().script_pubkey.0, - ":value_zat": &i64::from(output.txout().value), + ":value_zat": &i64::from(Amount::from(output.txout().value)), ":height": &u32::from(output.height()), ]; @@ -1708,7 +1711,7 @@ pub(crate) fn insert_sent_output( ":from_account": &u32::from(from_account), ":to_address": &to_address, ":to_account": &to_account, - ":value": &i64::from(output.value()), + ":value": &i64::from(Amount::from(output.value())), ":memo": memo_repr(output.memo()) ]; @@ -1736,7 +1739,7 @@ pub(crate) fn put_sent_output( tx_ref: i64, output_index: usize, recipient: &Recipient, - value: Amount, + value: NonNegativeAmount, memo: Option<&MemoBytes>, ) -> Result<(), SqliteClientError> { let mut stmt_upsert_sent_output = conn.prepare_cached( @@ -1762,7 +1765,7 @@ pub(crate) fn put_sent_output( ":from_account": &u32::from(from_account), ":to_address": &to_address, ":to_account": &to_account, - ":value": &i64::from(value), + ":value": &i64::from(Amount::from(value)), ":memo": memo_repr(memo) ]; @@ -2016,7 +2019,7 @@ mod tests { assert!(bal_absent.is_empty()); // Create a fake transparent output. - let value = Amount::from_u64(100000).unwrap(); + let value = NonNegativeAmount::const_from_u64(100000); let outpoint = OutPoint::new([1u8; 32], 1); let txout = TxOut { value, @@ -2064,7 +2067,7 @@ mod tests { assert_matches!( st.wallet().get_transparent_balances(account_id, height_2), - Ok(h) if h.get(taddr) == Some(&value) + Ok(h) if h.get(taddr) == Some(&value.into()) ); // Artificially delete the address from the addresses table so that @@ -2134,8 +2137,8 @@ mod tests { .unwrap() .into_iter() .map(|utxo| utxo.value()) - .sum::>(), - Some(Amount::from(expected)), + .sum::>(), + Some(expected), ); }; @@ -2147,7 +2150,7 @@ mod tests { let value = NonNegativeAmount::from_u64(100000).unwrap(); let outpoint = OutPoint::new([1u8; 32], 1); let txout = TxOut { - value: value.into(), + value, script_pubkey: taddr.script(), }; diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs index d0b422a022..5d838aae2a 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs @@ -396,6 +396,8 @@ mod tests { #[test] #[cfg(feature = "transparent-inputs")] fn migrate_from_wm2() { + use zcash_primitives::transaction::components::amount::NonNegativeAmount; + let network = Network::TestNetwork; let data_file = NamedTempFile::new().unwrap(); let mut db_data = WalletDb::for_path(data_file.path(), network).unwrap(); @@ -419,7 +421,7 @@ mod tests { sequence: 0, }], vout: vec![TxOut { - value: Amount::from_i64(1100000000).unwrap(), + value: NonNegativeAmount::const_from_u64(1100000000), script_pubkey: Script(vec![]), }], authorization: Authorized, diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 6072bb48cd..c41a914633 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -9,7 +9,10 @@ use zcash_primitives::{ consensus::BlockHeight, memo::MemoBytes, sapling::{self, Diversifier, Note, Nullifier, Rseed}, - transaction::{components::Amount, TxId}, + transaction::{ + components::{amount::NonNegativeAmount, Amount}, + TxId, + }, zip32::AccountId, }; @@ -97,7 +100,9 @@ fn to_spendable_note(row: &Row) -> Result, S Diversifier(tmp) }; - let note_value = Amount::from_i64(row.get(4)?).unwrap(); + let note_value = NonNegativeAmount::from_nonnegative_i64(row.get(4)?).map_err(|_e| { + SqliteClientError::CorruptedData("Note values must be nonnegative".to_string()) + })?; let rseed = { let rcm_bytes: Vec<_> = row.get(5)?; @@ -534,7 +539,7 @@ pub(crate) mod tests { let to: RecipientAddress = to_extsk.default_address().1.into(); let request = zip321::TransactionRequest::new(vec![Payment { recipient_address: to, - amount: Amount::const_from_i64(10000), + amount: NonNegativeAmount::const_from_u64(10000), memo: None, // this should result in the creation of an empty memo label: None, message: None, @@ -771,8 +776,8 @@ pub(crate) mod tests { available, required }) - if available == Amount::const_from_i64(50000) - && required == Amount::const_from_i64(80000) + if available == NonNegativeAmount::const_from_u64(50000) + && required == NonNegativeAmount::const_from_u64(80000) ); // Mine blocks SAPLING_ACTIVATION_HEIGHT + 2 to 9 until just before the second @@ -800,8 +805,8 @@ pub(crate) mod tests { available, required }) - if available == Amount::const_from_i64(50000) - && required == Amount::const_from_i64(80000) + if available == NonNegativeAmount::const_from_u64(50000) + && required == NonNegativeAmount::const_from_u64(80000) ); // Mine block 11 so that the second note becomes verified @@ -893,7 +898,7 @@ pub(crate) mod tests { available, required }) - if available == Amount::zero() && required == Amount::const_from_i64(12000) + if available == NonNegativeAmount::ZERO && required == NonNegativeAmount::const_from_u64(12000) ); // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 41 (that don't send us funds) @@ -922,7 +927,7 @@ pub(crate) mod tests { available, required }) - if available == Amount::zero() && required == Amount::const_from_i64(12000) + if available == NonNegativeAmount::ZERO && required == NonNegativeAmount::const_from_u64(12000) ); // Mine block SAPLING_ACTIVATION_HEIGHT + 42 so that the first transaction expires @@ -1180,7 +1185,7 @@ pub(crate) mod tests { // payment to an external recipient Payment { recipient_address: RecipientAddress::Shielded(addr2), - amount: amount_sent.into(), + amount: amount_sent, memo: None, label: None, message: None, @@ -1189,7 +1194,7 @@ pub(crate) mod tests { // payment back to the originating wallet, simulating legacy change Payment { recipient_address: RecipientAddress::Shielded(addr), - amount: amount_legacy_change.into(), + amount: amount_legacy_change, memo: None, label: None, message: None, @@ -1299,7 +1304,7 @@ pub(crate) mod tests { // This first request will fail due to insufficient non-dust funds let req = TransactionRequest::new(vec![Payment { recipient_address: RecipientAddress::Shielded(dfvk.default_address().1), - amount: Amount::const_from_i64(50000), + amount: NonNegativeAmount::const_from_u64(50000), memo: None, label: None, message: None, @@ -1316,15 +1321,15 @@ pub(crate) mod tests { NonZeroU32::new(1).unwrap(), ), Err(Error::InsufficientFunds { available, required }) - if available == Amount::const_from_i64(51000) - && required == Amount::const_from_i64(60000) + if available == NonNegativeAmount::const_from_u64(51000) + && required == NonNegativeAmount::const_from_u64(60000) ); // This request will succeed, spending a single dust input to pay the 10000 // ZAT fee in addition to the 41000 ZAT output to the recipient let req = TransactionRequest::new(vec![Payment { recipient_address: RecipientAddress::Shielded(dfvk.default_address().1), - amount: Amount::const_from_i64(41000), + amount: NonNegativeAmount::const_from_u64(41000), memo: None, label: None, message: None, @@ -1350,7 +1355,7 @@ pub(crate) mod tests { // in the total balance. assert_eq!( st.get_total_balance(account), - (total - NonNegativeAmount::from_u64(10000).unwrap()).unwrap() + (total - NonNegativeAmount::const_from_u64(10000)).unwrap() ); } @@ -1383,7 +1388,7 @@ pub(crate) mod tests { let utxo = WalletTransparentOutput::from_parts( OutPoint::new([1u8; 32], 1), TxOut { - value: Amount::const_from_i64(10000), + value: NonNegativeAmount::const_from_u64(10000), script_pubkey: taddr.script(), }, h, diff --git a/zcash_extensions/src/transparent/demo.rs b/zcash_extensions/src/transparent/demo.rs index 6935c8b72e..8dbe46f6b6 100644 --- a/zcash_extensions/src/transparent/demo.rs +++ b/zcash_extensions/src/transparent/demo.rs @@ -489,7 +489,7 @@ mod tests { transaction::{ builder::Builder, components::{ - amount::Amount, + amount::{Amount, NonNegativeAmount}, tze::{Authorized, Bundle, OutPoint, TzeIn, TzeOut}, }, fees::fixed, @@ -828,10 +828,10 @@ mod tests { .add_sapling_spend(extsk, *to.diversifier(), note1, witness1.path().unwrap()) .unwrap(); - let value = Amount::from_u64(100000).unwrap(); + let value = NonNegativeAmount::const_from_u64(100000); let (h1, h2) = demo_hashes(&preimage_1, &preimage_2); builder_a - .demo_open(value, h1) + .demo_open(value.into(), h1) .map_err(|e| format!("open failure: {:?}", e)) .unwrap(); let (tx_a, _) = builder_a @@ -847,9 +847,9 @@ mod tests { let mut builder_b = demo_builder(tx_height + 1); let prevout_a = (OutPoint::new(tx_a.txid(), 0), tze_a.vout[0].clone()); - let value_xfr = (value - fee_rule.fixed_fee().into()).unwrap(); + let value_xfr = (value - fee_rule.fixed_fee()).unwrap(); builder_b - .demo_transfer_to_close(prevout_a, value_xfr, preimage_1, h2) + .demo_transfer_to_close(prevout_a, value_xfr.into(), preimage_1, h2) .map_err(|e| format!("transfer failure: {:?}", e)) .unwrap(); let (tx_b, _) = builder_b @@ -873,7 +873,7 @@ mod tests { builder_c .add_transparent_output( &TransparentAddress::PublicKey([0; 20]), - (value_xfr - fee_rule.fixed_fee().into()).unwrap(), + (value_xfr - fee_rule.fixed_fee()).unwrap(), ) .unwrap(); diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 213f770938..4397c220cc 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -49,8 +49,25 @@ and this library adheres to Rust's notion of - `fees::fixed::FeeRule::fixed_fee` now wraps a `NonNegativeAmount` instead of an `Amount` - `fees::zip317::FeeRule::marginal_fee` is now represented and exposed as a `NonNegativeAmount` instead of an `Amount` +- `zcash_primitives::transaction::sighash::TransparentAuthorizingContext::input_amounts` now + returns the input values as `NonNegativeAmount` instead of as `Amount` - `zcash_primitives::transaction::components::sapling`: - `MapAuth` trait methods now take `&mut self` instead of `&self`. + - `sapling::fees::InputView::value` now returns a `NonNegativeAmount` instead of an `Amount` + - `sapling::fees::OutputView::value` now returns a `NonNegativeAmount` instead of an `Amount` +- `zcash_primitives::transaction::components::transparent`: + - `transparent::TxOut::value` now has type `NonNegativeAmount` instead of `Amount` + - `transparent::builder::TransparentBuilder::add_output` now takes its `value` + parameter as a `NonNegativeAmount` instead of as an `Amount`. + - `transparent::fees::InputView::value` now returns a `NonNegativeAmount` instead of an `Amount` + - `transparent::fees::OutputView::value` now returns a `NonNegativeAmount` instead of an `Amount` +- The following `zcash_primitives::transaction::builder::Builder` methods + have changed to take a `NonNegativeAmount` for their `value` arguments, + instead of an `Amount`. + - `Builder::add_sapling_output` + - `Builder::add_transparent_output` +- `zcash_primitives::transaction::components::amount::testing::arb_nonnegative_amount` + now returns a `NonNegativeAmount` instead of an `Amount` ### Removed - `zcash_primitives::constants`: diff --git a/zcash_primitives/src/transaction/builder.rs b/zcash_primitives/src/transaction/builder.rs index 24e09f084e..fba0bb0fb9 100644 --- a/zcash_primitives/src/transaction/builder.rs +++ b/zcash_primitives/src/transaction/builder.rs @@ -337,21 +337,14 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { &mut self, ovk: Option, to: PaymentAddress, - value: Amount, + value: NonNegativeAmount, memo: MemoBytes, ) -> Result<(), sapling_builder::Error> { - if value.is_negative() { - return Err(sapling_builder::Error::InvalidAmount); - } self.sapling_builder.add_output( &mut self.rng, ovk, to, - NoteValue::from_raw( - value - .try_into() - .expect("Cannot create Sapling outputs with negative note values."), - ), + NoteValue::from_raw(value.into()), memo, ) } @@ -372,7 +365,7 @@ impl<'a, P: consensus::Parameters, R: RngCore + CryptoRng> Builder<'a, P, R> { pub fn add_transparent_output( &mut self, to: &TransparentAddress, - value: Amount, + value: NonNegativeAmount, ) -> Result<(), transparent::builder::Error> { self.transparent_builder.add_output(to, value) } @@ -720,7 +713,6 @@ mod tests { transaction::components::{ amount::{Amount, NonNegativeAmount}, sapling::builder::{self as sapling_builder}, - transparent::builder::{self as transparent_builder}, }, zip32::ExtendedSpendingKey, }; @@ -741,29 +733,6 @@ mod tests { zip32::AccountId, }; - #[test] - fn fails_on_negative_output() { - let extsk = ExtendedSpendingKey::master(&[]); - let dfvk = extsk.to_diversifiable_full_viewing_key(); - let ovk = dfvk.fvk().ovk; - let to = dfvk.default_address().1; - - let sapling_activation_height = TEST_NETWORK - .activation_height(NetworkUpgrade::Sapling) - .unwrap(); - - let mut builder = Builder::new(TEST_NETWORK, sapling_activation_height, None); - assert_eq!( - builder.add_sapling_output( - Some(ovk), - to, - Amount::from_i64(-1).unwrap(), - MemoBytes::empty() - ), - Err(sapling_builder::Error::InvalidAmount) - ); - } - // This test only works with the transparent_inputs feature because we have to // be able to create a tx with a valid balance, without using Sapling inputs. #[test] @@ -795,7 +764,7 @@ mod tests { let tsk = AccountPrivKey::from_seed(&TEST_NETWORK, &[0u8; 32], AccountId::from(0)).unwrap(); let prev_coin = TxOut { - value: Amount::from_u64(50000).unwrap(), + value: NonNegativeAmount::const_from_u64(50000), script_pubkey: tsk .to_account_pubkey() .derive_external_ivk() @@ -816,7 +785,7 @@ mod tests { builder .add_transparent_output( &TransparentAddress::PublicKey([0; 20]), - Amount::from_u64(40000).unwrap(), + NonNegativeAmount::const_from_u64(40000), ) .unwrap(); @@ -852,7 +821,7 @@ mod tests { builder .add_transparent_output( &TransparentAddress::PublicKey([0; 20]), - Amount::from_u64(40000).unwrap(), + NonNegativeAmount::const_from_u64(40000), ) .unwrap(); @@ -864,21 +833,6 @@ mod tests { ); } - #[test] - fn fails_on_negative_transparent_output() { - let tx_height = TEST_NETWORK - .activation_height(NetworkUpgrade::Sapling) - .unwrap(); - let mut builder = Builder::new(TEST_NETWORK, tx_height, None); - assert_eq!( - builder.add_transparent_output( - &TransparentAddress::PublicKey([0; 20]), - Amount::from_i64(-1).unwrap(), - ), - Err(transparent_builder::Error::InvalidAmount) - ); - } - #[test] fn fails_on_negative_change() { use crate::transaction::fees::zip317::MINIMUM_FEE; @@ -913,7 +867,7 @@ mod tests { .add_sapling_output( ovk, to, - Amount::from_u64(50000).unwrap(), + NonNegativeAmount::const_from_u64(50000), MemoBytes::empty(), ) .unwrap(); @@ -931,7 +885,7 @@ mod tests { builder .add_transparent_output( &TransparentAddress::PublicKey([0; 20]), - Amount::from_u64(50000).unwrap(), + NonNegativeAmount::const_from_u64(50000), ) .unwrap(); assert_matches!( @@ -963,14 +917,14 @@ mod tests { .add_sapling_output( ovk, to, - Amount::from_u64(30000).unwrap(), + NonNegativeAmount::const_from_u64(30000), MemoBytes::empty(), ) .unwrap(); builder .add_transparent_output( &TransparentAddress::PublicKey([0; 20]), - Amount::from_u64(20000).unwrap(), + NonNegativeAmount::const_from_u64(20000), ) .unwrap(); assert_matches!( @@ -1007,14 +961,14 @@ mod tests { .add_sapling_output( ovk, to, - Amount::from_u64(30000).unwrap(), + NonNegativeAmount::const_from_u64(30000), MemoBytes::empty(), ) .unwrap(); builder .add_transparent_output( &TransparentAddress::PublicKey([0; 20]), - Amount::from_u64(20000).unwrap(), + NonNegativeAmount::const_from_u64(20000), ) .unwrap(); assert_matches!( diff --git a/zcash_primitives/src/transaction/components/amount.rs b/zcash_primitives/src/transaction/components/amount.rs index 996b2e7bc8..8c4d6fc641 100644 --- a/zcash_primitives/src/transaction/components/amount.rs +++ b/zcash_primitives/src/transaction/components/amount.rs @@ -281,6 +281,19 @@ impl NonNegativeAmount { let amount = u64::from_le_bytes(bytes); Self::from_u64(amount) } + + /// Reads a NonNegativeAmount from a signed 64-bit little-endian integer. + /// + /// Returns an error if the amount is outside the range `{0..MAX_MONEY}`. + pub fn from_nonnegative_i64_le_bytes(bytes: [u8; 8]) -> Result { + let amount = i64::from_le_bytes(bytes); + Self::from_nonnegative_i64(amount) + } + + /// Returns this NonNegativeAmount encoded as a signed 64-bit little-endian integer. + pub fn to_i64_le_bytes(self) -> [u8; 8] { + self.0.to_i64_le_bytes() + } } impl From for Amount { @@ -400,7 +413,7 @@ impl std::fmt::Display for BalanceError { pub mod testing { use proptest::prelude::prop_compose; - use super::{Amount, MAX_MONEY}; + use super::{Amount, NonNegativeAmount, MAX_MONEY}; prop_compose! { pub fn arb_amount()(amt in -MAX_MONEY..MAX_MONEY) -> Amount { @@ -409,8 +422,8 @@ pub mod testing { } prop_compose! { - pub fn arb_nonnegative_amount()(amt in 0i64..MAX_MONEY) -> Amount { - Amount::from_i64(amt).unwrap() + pub fn arb_nonnegative_amount()(amt in 0i64..MAX_MONEY) -> NonNegativeAmount { + NonNegativeAmount::from_u64(amt as u64).unwrap() } } diff --git a/zcash_primitives/src/transaction/components/sapling/builder.rs b/zcash_primitives/src/transaction/components/sapling/builder.rs index 7890d5143c..8687ccb2ba 100644 --- a/zcash_primitives/src/transaction/components/sapling/builder.rs +++ b/zcash_primitives/src/transaction/components/sapling/builder.rs @@ -23,7 +23,7 @@ use crate::{ transaction::{ builder::Progress, components::{ - amount::Amount, + amount::{Amount, NonNegativeAmount}, sapling::{ fees, Authorization, Authorized, Bundle, GrothProofBytes, OutputDescription, SpendDescription, @@ -75,9 +75,9 @@ impl fees::InputView<()> for SpendDescriptionInfo { &() } - fn value(&self) -> Amount { + fn value(&self) -> NonNegativeAmount { // An existing note to be spent must have a valid amount value. - Amount::from_u64(self.note.value().inner()).unwrap() + NonNegativeAmount::from_u64(self.note.value().inner()).unwrap() } } @@ -144,8 +144,8 @@ impl SaplingOutputInfo { } impl fees::OutputView for SaplingOutputInfo { - fn value(&self) -> Amount { - Amount::from_u64(self.note.value().inner()) + fn value(&self) -> NonNegativeAmount { + NonNegativeAmount::from_u64(self.note.value().inner()) .expect("Note values should be checked at construction.") } } diff --git a/zcash_primitives/src/transaction/components/sapling/fees.rs b/zcash_primitives/src/transaction/components/sapling/fees.rs index 10d72adb60..61800b1e82 100644 --- a/zcash_primitives/src/transaction/components/sapling/fees.rs +++ b/zcash_primitives/src/transaction/components/sapling/fees.rs @@ -1,7 +1,7 @@ //! Types related to computation of fees and change related to the Sapling components //! of a transaction. -use crate::transaction::components::amount::Amount; +use crate::transaction::components::amount::NonNegativeAmount; /// A trait that provides a minimized view of a Sapling input suitable for use in /// fee and change calculation. @@ -9,12 +9,12 @@ pub trait InputView { /// An identifier for the input being spent. fn note_id(&self) -> &NoteRef; /// The value of the input being spent. - fn value(&self) -> Amount; + fn value(&self) -> NonNegativeAmount; } /// A trait that provides a minimized view of a Sapling output suitable for use in /// fee and change calculation. pub trait OutputView { /// The value of the output being produced. - fn value(&self) -> Amount; + fn value(&self) -> NonNegativeAmount; } diff --git a/zcash_primitives/src/transaction/components/transparent.rs b/zcash_primitives/src/transaction/components/transparent.rs index 988174e2f9..e580f40501 100644 --- a/zcash_primitives/src/transaction/components/transparent.rs +++ b/zcash_primitives/src/transaction/components/transparent.rs @@ -7,7 +7,7 @@ use std::io::{self, Read, Write}; use crate::legacy::{Script, TransparentAddress}; -use super::amount::{Amount, BalanceError}; +use super::amount::{Amount, BalanceError, NonNegativeAmount}; pub mod builder; pub mod fees; @@ -82,7 +82,7 @@ impl Bundle { let output_sum = self .vout .iter() - .map(|p| p.value) + .map(|p| Amount::from(p.value)) .sum::>() .ok_or(BalanceError::Overflow)?; @@ -159,7 +159,7 @@ impl TxIn { #[derive(Clone, Debug, PartialEq, Eq)] pub struct TxOut { - pub value: Amount, + pub value: NonNegativeAmount, pub script_pubkey: Script, } @@ -168,7 +168,7 @@ impl TxOut { let value = { let mut tmp = [0u8; 8]; reader.read_exact(&mut tmp)?; - Amount::from_nonnegative_i64_le_bytes(tmp) + NonNegativeAmount::from_nonnegative_i64_le_bytes(tmp) } .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "value out of range"))?; let script_pubkey = Script::read(&mut reader)?; diff --git a/zcash_primitives/src/transaction/components/transparent/builder.rs b/zcash_primitives/src/transaction/components/transparent/builder.rs index 95a8b89491..757f8c8c44 100644 --- a/zcash_primitives/src/transaction/components/transparent/builder.rs +++ b/zcash_primitives/src/transaction/components/transparent/builder.rs @@ -6,7 +6,7 @@ use crate::{ legacy::{Script, TransparentAddress}, transaction::{ components::{ - amount::{Amount, BalanceError}, + amount::{Amount, BalanceError, NonNegativeAmount}, transparent::{self, fees, Authorization, Authorized, Bundle, TxIn, TxOut}, }, sighash::TransparentAuthorizingContext, @@ -134,10 +134,6 @@ impl TransparentBuilder { utxo: OutPoint, coin: TxOut, ) -> Result<(), Error> { - if coin.value.is_negative() { - return Err(Error::InvalidAmount); - } - // Ensure that the RIPEMD-160 digest of the public key associated with the // provided secret key matches that of the address to which the provided // output may be spent. @@ -164,11 +160,11 @@ impl TransparentBuilder { Ok(()) } - pub fn add_output(&mut self, to: &TransparentAddress, value: Amount) -> Result<(), Error> { - if value.is_negative() { - return Err(Error::InvalidAmount); - } - + pub fn add_output( + &mut self, + to: &TransparentAddress, + value: NonNegativeAmount, + ) -> Result<(), Error> { self.vout.push(TxOut { value, script_pubkey: to.script(), @@ -183,20 +179,20 @@ impl TransparentBuilder { .inputs .iter() .map(|input| input.coin.value) - .sum::>() + .sum::>() .ok_or(BalanceError::Overflow)?; #[cfg(not(feature = "transparent-inputs"))] - let input_sum = Amount::zero(); + let input_sum = NonNegativeAmount::ZERO; let output_sum = self .vout .iter() .map(|vo| vo.value) - .sum::>() + .sum::>() .ok_or(BalanceError::Overflow)?; - (input_sum - output_sum).ok_or(BalanceError::Underflow) + (Amount::from(input_sum) - Amount::from(output_sum)).ok_or(BalanceError::Underflow) } pub fn build(self) -> Option> { @@ -241,7 +237,7 @@ impl TxIn { #[cfg(not(feature = "transparent-inputs"))] impl TransparentAuthorizingContext for Unauthorized { - fn input_amounts(&self) -> Vec { + fn input_amounts(&self) -> Vec { vec![] } @@ -252,7 +248,7 @@ impl TransparentAuthorizingContext for Unauthorized { #[cfg(feature = "transparent-inputs")] impl TransparentAuthorizingContext for Unauthorized { - fn input_amounts(&self) -> Vec { + fn input_amounts(&self) -> Vec { return self.inputs.iter().map(|txin| txin.coin.value).collect(); } diff --git a/zcash_primitives/src/transaction/components/transparent/fees.rs b/zcash_primitives/src/transaction/components/transparent/fees.rs index 4b3f4ddb01..12ac0d6160 100644 --- a/zcash_primitives/src/transaction/components/transparent/fees.rs +++ b/zcash_primitives/src/transaction/components/transparent/fees.rs @@ -4,7 +4,7 @@ use super::TxOut; use crate::{ legacy::Script, - transaction::{components::amount::Amount, OutPoint}, + transaction::{components::amount::NonNegativeAmount, OutPoint}, }; /// This trait provides a minimized view of a transparent input suitable for use in @@ -20,13 +20,13 @@ pub trait InputView: std::fmt::Debug { /// fee and change computation. pub trait OutputView: std::fmt::Debug { /// Returns the value of the output being created. - fn value(&self) -> Amount; + fn value(&self) -> NonNegativeAmount; /// Returns the script corresponding to the newly created output. fn script_pubkey(&self) -> &Script; } impl OutputView for TxOut { - fn value(&self) -> Amount { + fn value(&self) -> NonNegativeAmount { self.value } diff --git a/zcash_primitives/src/transaction/components/tze.rs b/zcash_primitives/src/transaction/components/tze.rs index e766d2f9f5..e038d3eec9 100644 --- a/zcash_primitives/src/transaction/components/tze.rs +++ b/zcash_primitives/src/transaction/components/tze.rs @@ -253,7 +253,7 @@ pub mod testing { prop_compose! { pub fn arb_tzeout()(value in arb_nonnegative_amount(), precondition in arb_precondition()) -> TzeOut { - TzeOut { value, precondition } + TzeOut { value: value.into(), precondition } } } diff --git a/zcash_primitives/src/transaction/sighash.rs b/zcash_primitives/src/transaction/sighash.rs index 3fd9b8b2ea..9ab8bf4090 100644 --- a/zcash_primitives/src/transaction/sighash.rs +++ b/zcash_primitives/src/transaction/sighash.rs @@ -3,6 +3,7 @@ use blake2b_simd::Hash as Blake2bHash; use super::{ components::{ + amount::NonNegativeAmount, sapling::{self, GrothProofBytes}, transparent, Amount, }, @@ -27,7 +28,7 @@ pub enum SignableInput<'a> { index: usize, script_code: &'a Script, script_pubkey: &'a Script, - value: Amount, + value: NonNegativeAmount, }, #[cfg(feature = "zfuture")] Tze { @@ -63,7 +64,7 @@ pub trait TransparentAuthorizingContext: transparent::Authorization { /// so that wallets can commit to the transparent input breakdown /// without requiring the full data of the previous transactions /// providing these inputs. - fn input_amounts(&self) -> Vec; + fn input_amounts(&self) -> Vec; /// Returns the list of all transparent input scriptPubKeys, provided /// so that wallets can commit to the transparent input breakdown /// without requiring the full data of the previous transactions diff --git a/zcash_primitives/src/transaction/tests.rs b/zcash_primitives/src/transaction/tests.rs index 9335d89840..44c0534bc8 100644 --- a/zcash_primitives/src/transaction/tests.rs +++ b/zcash_primitives/src/transaction/tests.rs @@ -3,10 +3,11 @@ use std::ops::Deref; use proptest::prelude::*; -use crate::{consensus::BranchId, legacy::Script}; +use crate::{ + consensus::BranchId, legacy::Script, transaction::components::amount::NonNegativeAmount, +}; use super::{ - components::Amount, sapling, sighash::{ SignableInput, TransparentAuthorizingContext, SIGHASH_ALL, SIGHASH_ANYONECANPAY, @@ -134,7 +135,7 @@ fn zip_0143() { index: n as usize, script_code: &tv.script_code, script_pubkey: &tv.script_code, - value: Amount::from_nonnegative_i64(tv.amount).unwrap(), + value: NonNegativeAmount::from_nonnegative_i64(tv.amount).unwrap(), }, _ => SignableInput::Shielded, }; @@ -156,7 +157,7 @@ fn zip_0243() { index: n as usize, script_code: &tv.script_code, script_pubkey: &tv.script_code, - value: Amount::from_nonnegative_i64(tv.amount).unwrap(), + value: NonNegativeAmount::from_nonnegative_i64(tv.amount).unwrap(), }, _ => SignableInput::Shielded, }; @@ -170,7 +171,7 @@ fn zip_0243() { #[derive(Debug)] struct TestTransparentAuth { - input_amounts: Vec, + input_amounts: Vec, input_scriptpubkeys: Vec