From 07ebc404c70f83e8251f4563d99d98ed3bdb35de Mon Sep 17 00:00:00 2001 From: borngraced Date: Mon, 14 Aug 2023 11:20:06 +0100 Subject: [PATCH 01/42] =?UTF-8?q?save=20dev=20state=20=E2=80=94=20added=20?= =?UTF-8?q?mm2=5Fext=20modules=20to=20sqlite=5Fclient=20and=20backend?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- zcash_client_backend/Cargo.toml | 1 + zcash_client_backend/src/lib.rs | 1 + zcash_client_backend/src/mm2_ext/data_api.rs | 217 +++++++++++++++++++ zcash_client_backend/src/mm2_ext/mod.rs | 1 + zcash_client_sqlite/src/lib.rs | 1 + zcash_client_sqlite/src/mm2_ext/mod.rs | 0 6 files changed, 221 insertions(+) create mode 100644 zcash_client_backend/src/mm2_ext/data_api.rs create mode 100644 zcash_client_backend/src/mm2_ext/mod.rs create mode 100644 zcash_client_sqlite/src/mm2_ext/mod.rs diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index b169ba2d12..d423d2819b 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -18,6 +18,7 @@ bls12_381 = "0.3.1" bs58 = { version = "0.4", features = ["check"] } base64 = "0.13" ff = "0.8" +futures01 = { version = "0.1", package = "futures" } group = "0.8" hex = "0.4" jubjub = "0.5.1" diff --git a/zcash_client_backend/src/lib.rs b/zcash_client_backend/src/lib.rs index 085070c134..b416ec897f 100644 --- a/zcash_client_backend/src/lib.rs +++ b/zcash_client_backend/src/lib.rs @@ -13,6 +13,7 @@ pub mod data_api; mod decrypt; pub mod encoding; pub mod keys; +pub mod mm2_ext; pub mod proto; pub mod wallet; pub mod welding_rig; diff --git a/zcash_client_backend/src/mm2_ext/data_api.rs b/zcash_client_backend/src/mm2_ext/data_api.rs new file mode 100644 index 0000000000..737b5cc99b --- /dev/null +++ b/zcash_client_backend/src/mm2_ext/data_api.rs @@ -0,0 +1,217 @@ +use crate::proto::compact_formats::CompactBlock; +use std::fmt::Debug; +use std::sync::{Arc, Mutex}; + +use crate::data_api::error::{ChainInvalid, Error}; +use crate::data_api::{PrunedBlock, WalletWrite}; +use crate::wallet::{AccountId, WalletTx}; +use crate::welding_rig::scan_block; +use futures01::Future; +use zcash_primitives::block::BlockHash; +use zcash_primitives::consensus; +use zcash_primitives::consensus::{BlockHeight, NetworkUpgrade}; +use zcash_primitives::merkle_tree::CommitmentTree; +use zcash_primitives::sapling::Nullifier; +use zcash_primitives::zip32::ExtendedFullViewingKey; + +#[macro_export] +macro_rules! try_f { + ($e: expr) => { + match $e { + Ok(ok) => ok, + Err(e) => return Box::new(futures01::future::err(e.into())), + } + }; +} + +/// This trait provides sequential access to raw blockchain data via a callback-oriented +/// API. +pub trait BlockSource { + type Error; + + /// Scan the specified `limit` number of blocks from the blockchain, starting at + /// `from_height`, applying the provided callback to each block. + fn with_blocks( + &self, + from_height: BlockHeight, + limit: Option, + with_row: Box, + ) -> Box + Send> + where + F: FnMut(CompactBlock) -> Result<(), Self::Error>; +} + +pub fn validate_chain<'a, N, E, P, C>( + parameters: &P, + cache: &C, + validate_from: Option<(BlockHeight, BlockHash)>, +) -> Box + Send + 'a> +where + E: From> + Send + 'a, + P: consensus::Parameters, + C: BlockSource, +{ + let sapling_activation_height = try_f!(parameters + .activation_height(NetworkUpgrade::Sapling) + .ok_or(Error::SaplingNotActive)); + + // The cache will contain blocks above the `validate_from` height. Validate from that maximum + // height up to the chain tip, returning the hash of the block found in the cache at the + // `validate_from` height, which can then be used to verify chain integrity by comparing + // against the `validate_from` hash. + let from_height = validate_from + .map(|(height, _)| height) + .unwrap_or(sapling_activation_height - 1); + + let mut prev_height = from_height; + let mut prev_hash: Option = validate_from.map(|(_, hash)| hash); + + cache.with_blocks( + from_height, + None, + Box::new(|block: CompactBlock| { + let current_height = block.height(); + let result = if current_height != prev_height + 1 { + Err(ChainInvalid::block_height_discontinuity( + prev_height + 1, + current_height, + )) + } else { + match prev_hash { + None => Ok(()), + Some(h) if h == block.prev_hash() => Ok(()), + Some(_) => Err(ChainInvalid::prev_hash_mismatch(current_height)), + } + }; + + prev_height = current_height; + prev_hash = Some(block.hash()); + result.map_err(E::from) + }), + ) +} + +pub fn scan_cached_blocks<'a, E, N, P, C, D>( + params: &P, + cache: &C, + data: Arc>, + limit: Option, +) -> Box + Send + 'a> +where + P: consensus::Parameters + Send + Sync, + C: BlockSource, + D: WalletWrite, + N: Copy + Debug + Send, + E: From> + Send + 'a, +{ + let mut data_guard = data.lock().unwrap(); + let sapling_activation_height = try_f!(params + .activation_height(NetworkUpgrade::Sapling) + .ok_or(Error::SaplingNotActive)); + + // Recall where we synced up to previously. + // If we have never synced, use sapling activation height to select all cached CompactBlocks. + let mut last_height = try_f!(data_guard.block_height_extrema().map(|opt| { + opt.map(|(_, max)| max) + .unwrap_or(sapling_activation_height - 1) + })); + + // Fetch the ExtendedFullViewingKeys we are tracking + let extfvks = try_f!(data_guard.get_extended_full_viewing_keys()); + let extfvks: Vec<(&AccountId, &ExtendedFullViewingKey)> = extfvks.iter().collect(); + + // Get the most recent CommitmentTree + let mut tree = try_f!(data_guard + .get_commitment_tree(last_height) + .map(|t| t.unwrap_or_else(CommitmentTree::empty))); + + // Get most recent incremental witnesses for the notes we are tracking + let mut witnesses = try_f!(data_guard.get_witnesses(last_height)); + + // Get the nullifiers for the notes we are tracking + let mut nullifiers = try_f!(data_guard.get_nullifiers()); + + cache.with_blocks( + last_height, + limit, + Box::new(|block: CompactBlock| { + let current_height = block.height(); + + // Scanned blocks MUST be height-sequential. + if current_height != (last_height + 1) { + return Err(ChainInvalid::block_height_discontinuity( + last_height + 1, + current_height, + ) + .into()); + } + + let block_hash = BlockHash::from_slice(&block.hash); + let block_time = block.time; + + let txs: Vec> = { + let mut witness_refs: Vec<_> = witnesses.iter_mut().map(|w| &mut w.1).collect(); + + scan_block( + params, + block, + &extfvks, + &nullifiers, + &mut tree, + &mut witness_refs[..], + ) + }; + + // Enforce that all roots match. This is slow, so only include in debug builds. + #[cfg(debug_assertions)] + { + let cur_root = tree.root(); + for row in &witnesses { + if row.1.root() != cur_root { + return Err(Error::InvalidWitnessAnchor(row.0, current_height).into()); + } + } + for tx in &txs { + for output in tx.shielded_outputs.iter() { + if output.witness.root() != cur_root { + return Err(Error::InvalidNewWitnessAnchor( + output.index, + tx.txid, + current_height, + output.witness.root(), + ) + .into()); + } + } + } + } + + let new_witnesses = data_guard.advance_by_block( + &(PrunedBlock { + block_height: current_height, + block_hash, + block_time, + commitment_tree: &tree, + transactions: &txs, + }), + &witnesses, + )?; + + let spent_nf: Vec = txs + .iter() + .flat_map(|tx| tx.shielded_spends.iter().map(|spend| spend.nf)) + .collect(); + nullifiers.retain(|(_, nf)| !spent_nf.contains(nf)); + nullifiers.extend( + txs.iter() + .flat_map(|tx| tx.shielded_outputs.iter().map(|out| (out.account, out.nf))), + ); + + witnesses.extend(new_witnesses); + + last_height = current_height; + + Ok(()) + }), + ) +} diff --git a/zcash_client_backend/src/mm2_ext/mod.rs b/zcash_client_backend/src/mm2_ext/mod.rs new file mode 100644 index 0000000000..c86bf429e0 --- /dev/null +++ b/zcash_client_backend/src/mm2_ext/mod.rs @@ -0,0 +1 @@ +pub mod data_api; diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index b2f4218a53..6c8d914ba3 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -61,6 +61,7 @@ use crate::error::SqliteClientError; pub mod chain; pub mod error; +pub mod mm2_ext; pub mod wallet; /// A newtype wrapper for sqlite primary key values for the notes diff --git a/zcash_client_sqlite/src/mm2_ext/mod.rs b/zcash_client_sqlite/src/mm2_ext/mod.rs new file mode 100644 index 0000000000..e69de29bb2 From d1158fe1ae37d0c117f7e3ccf5f7f403270bc383 Mon Sep 17 00:00:00 2001 From: borngraced Date: Tue, 22 Aug 2023 12:02:47 +0100 Subject: [PATCH 02/42] =?UTF-8?q?save=20dev=20state=20=E2=80=94=20create?= =?UTF-8?q?=20async=20fn=20version=20of=20data=5Fapi=20functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- zcash_client_backend/Cargo.toml | 2 +- zcash_client_backend/src/lib.rs | 2 +- .../src/{mm2_ext => with_async}/data_api.rs | 44 +++++++------------ .../src/{mm2_ext => with_async}/mod.rs | 0 zcash_client_sqlite/src/lib.rs | 2 +- .../src/{mm2_ext => with_async}/mod.rs | 0 6 files changed, 20 insertions(+), 30 deletions(-) rename zcash_client_backend/src/{mm2_ext => with_async}/data_api.rs (85%) rename zcash_client_backend/src/{mm2_ext => with_async}/mod.rs (100%) rename zcash_client_sqlite/src/{mm2_ext => with_async}/mod.rs (100%) diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index d423d2819b..ba7f0af7a2 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -13,12 +13,12 @@ license = "MIT OR Apache-2.0" edition = "2018" [dependencies] +async-trait = "0.1.52" bech32 = "0.9.1" bls12_381 = "0.3.1" bs58 = { version = "0.4", features = ["check"] } base64 = "0.13" ff = "0.8" -futures01 = { version = "0.1", package = "futures" } group = "0.8" hex = "0.4" jubjub = "0.5.1" diff --git a/zcash_client_backend/src/lib.rs b/zcash_client_backend/src/lib.rs index b416ec897f..011e329c0c 100644 --- a/zcash_client_backend/src/lib.rs +++ b/zcash_client_backend/src/lib.rs @@ -13,10 +13,10 @@ pub mod data_api; mod decrypt; pub mod encoding; pub mod keys; -pub mod mm2_ext; pub mod proto; pub mod wallet; pub mod welding_rig; +pub mod with_async; pub mod zip321; pub use decrypt::{decrypt_transaction, DecryptedOutput}; diff --git a/zcash_client_backend/src/mm2_ext/data_api.rs b/zcash_client_backend/src/with_async/data_api.rs similarity index 85% rename from zcash_client_backend/src/mm2_ext/data_api.rs rename to zcash_client_backend/src/with_async/data_api.rs index 737b5cc99b..5cab173bdb 100644 --- a/zcash_client_backend/src/mm2_ext/data_api.rs +++ b/zcash_client_backend/src/with_async/data_api.rs @@ -6,7 +6,6 @@ use crate::data_api::error::{ChainInvalid, Error}; use crate::data_api::{PrunedBlock, WalletWrite}; use crate::wallet::{AccountId, WalletTx}; use crate::welding_rig::scan_block; -use futures01::Future; use zcash_primitives::block::BlockHash; use zcash_primitives::consensus; use zcash_primitives::consensus::{BlockHeight, NetworkUpgrade}; @@ -14,18 +13,9 @@ use zcash_primitives::merkle_tree::CommitmentTree; use zcash_primitives::sapling::Nullifier; use zcash_primitives::zip32::ExtendedFullViewingKey; -#[macro_export] -macro_rules! try_f { - ($e: expr) => { - match $e { - Ok(ok) => ok, - Err(e) => return Box::new(futures01::future::err(e.into())), - } - }; -} - /// This trait provides sequential access to raw blockchain data via a callback-oriented /// API. +#[async_trait::async_trait] pub trait BlockSource { type Error; @@ -36,24 +26,24 @@ pub trait BlockSource { from_height: BlockHeight, limit: Option, with_row: Box, - ) -> Box + Send> + ) -> Result<(), Self::Error> where F: FnMut(CompactBlock) -> Result<(), Self::Error>; } -pub fn validate_chain<'a, N, E, P, C>( +pub async fn validate_chain<'a, N, E, P, C>( parameters: &P, cache: &C, validate_from: Option<(BlockHeight, BlockHash)>, -) -> Box + Send + 'a> +) -> Result<(), E> where E: From> + Send + 'a, P: consensus::Parameters, C: BlockSource, { - let sapling_activation_height = try_f!(parameters + let sapling_activation_height = parameters .activation_height(NetworkUpgrade::Sapling) - .ok_or(Error::SaplingNotActive)); + .ok_or(Error::SaplingNotActive)?; // The cache will contain blocks above the `validate_from` height. Validate from that maximum // height up to the chain tip, returning the hash of the block found in the cache at the @@ -91,12 +81,12 @@ where ) } -pub fn scan_cached_blocks<'a, E, N, P, C, D>( +pub async fn scan_cached_blocks<'a, E, N, P, C, D>( params: &P, cache: &C, data: Arc>, limit: Option, -) -> Box + Send + 'a> +) -> Result<(), E> where P: consensus::Parameters + Send + Sync, C: BlockSource, @@ -105,31 +95,31 @@ where E: From> + Send + 'a, { let mut data_guard = data.lock().unwrap(); - let sapling_activation_height = try_f!(params + let sapling_activation_height = params .activation_height(NetworkUpgrade::Sapling) - .ok_or(Error::SaplingNotActive)); + .ok_or(Error::SaplingNotActive)?; // Recall where we synced up to previously. // If we have never synced, use sapling activation height to select all cached CompactBlocks. - let mut last_height = try_f!(data_guard.block_height_extrema().map(|opt| { + let mut last_height = data_guard.block_height_extrema().map(|opt| { opt.map(|(_, max)| max) .unwrap_or(sapling_activation_height - 1) - })); + })?; // Fetch the ExtendedFullViewingKeys we are tracking - let extfvks = try_f!(data_guard.get_extended_full_viewing_keys()); + let extfvks = data_guard.get_extended_full_viewing_keys()?; let extfvks: Vec<(&AccountId, &ExtendedFullViewingKey)> = extfvks.iter().collect(); // Get the most recent CommitmentTree - let mut tree = try_f!(data_guard + let mut tree = data_guard .get_commitment_tree(last_height) - .map(|t| t.unwrap_or_else(CommitmentTree::empty))); + .map(|t| t.unwrap_or_else(CommitmentTree::empty))?; // Get most recent incremental witnesses for the notes we are tracking - let mut witnesses = try_f!(data_guard.get_witnesses(last_height)); + let mut witnesses = data_guard.get_witnesses(last_height)?; // Get the nullifiers for the notes we are tracking - let mut nullifiers = try_f!(data_guard.get_nullifiers()); + let mut nullifiers = data_guard.get_nullifiers()?; cache.with_blocks( last_height, diff --git a/zcash_client_backend/src/mm2_ext/mod.rs b/zcash_client_backend/src/with_async/mod.rs similarity index 100% rename from zcash_client_backend/src/mm2_ext/mod.rs rename to zcash_client_backend/src/with_async/mod.rs diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 6c8d914ba3..ad3b5a8d1a 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -61,8 +61,8 @@ use crate::error::SqliteClientError; pub mod chain; pub mod error; -pub mod mm2_ext; pub mod wallet; +pub mod with_async; /// A newtype wrapper for sqlite primary key values for the notes /// table. diff --git a/zcash_client_sqlite/src/mm2_ext/mod.rs b/zcash_client_sqlite/src/with_async/mod.rs similarity index 100% rename from zcash_client_sqlite/src/mm2_ext/mod.rs rename to zcash_client_sqlite/src/with_async/mod.rs From 8819ec2bbc66c515f64aecc89b2de856e87d320a Mon Sep 17 00:00:00 2001 From: borngraced Date: Tue, 22 Aug 2023 12:29:47 +0100 Subject: [PATCH 03/42] =?UTF-8?q?save=20dev=20state=20=E2=80=94=20modified?= =?UTF-8?q?=20async=20BlockSource=20trait?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/with_async/data_api.rs | 231 ++++++++++-------- 1 file changed, 127 insertions(+), 104 deletions(-) diff --git a/zcash_client_backend/src/with_async/data_api.rs b/zcash_client_backend/src/with_async/data_api.rs index 5cab173bdb..9ffaa04054 100644 --- a/zcash_client_backend/src/with_async/data_api.rs +++ b/zcash_client_backend/src/with_async/data_api.rs @@ -15,13 +15,13 @@ use zcash_primitives::zip32::ExtendedFullViewingKey; /// This trait provides sequential access to raw blockchain data via a callback-oriented /// API. -#[async_trait::async_trait] +#[async_trait::async_trait(?Send)] pub trait BlockSource { type Error; /// Scan the specified `limit` number of blocks from the blockchain, starting at /// `from_height`, applying the provided callback to each block. - fn with_blocks( + async fn with_blocks( &self, from_height: BlockHeight, limit: Option, @@ -31,6 +31,25 @@ pub trait BlockSource { F: FnMut(CompactBlock) -> Result<(), Self::Error>; } +struct BlockSourceCaged; + +#[async_trait::async_trait(?Send)] +impl BlockSource for BlockSourceCaged { + type Error = String; + + async fn with_blocks( + &self, + from_height: BlockHeight, + limit: Option, + with_row: Box, + ) -> Result<(), Self::Error> + where + F: FnMut(CompactBlock) -> Result<(), Self::Error>, + { + todo!() + } +} + pub async fn validate_chain<'a, N, E, P, C>( parameters: &P, cache: &C, @@ -56,29 +75,31 @@ where let mut prev_height = from_height; let mut prev_hash: Option = validate_from.map(|(_, hash)| hash); - cache.with_blocks( - from_height, - None, - Box::new(|block: CompactBlock| { - let current_height = block.height(); - let result = if current_height != prev_height + 1 { - Err(ChainInvalid::block_height_discontinuity( - prev_height + 1, - current_height, - )) - } else { - match prev_hash { - None => Ok(()), - Some(h) if h == block.prev_hash() => Ok(()), - Some(_) => Err(ChainInvalid::prev_hash_mismatch(current_height)), - } - }; - - prev_height = current_height; - prev_hash = Some(block.hash()); - result.map_err(E::from) - }), - ) + cache + .with_blocks( + from_height, + None, + Box::new(|block: CompactBlock| { + let current_height = block.height(); + let result = if current_height != prev_height + 1 { + Err(ChainInvalid::block_height_discontinuity( + prev_height + 1, + current_height, + )) + } else { + match prev_hash { + None => Ok(()), + Some(h) if h == block.prev_hash() => Ok(()), + Some(_) => Err(ChainInvalid::prev_hash_mismatch(current_height)), + } + }; + + prev_height = current_height; + prev_hash = Some(block.hash()); + result.map_err(E::from) + }), + ) + .await } pub async fn scan_cached_blocks<'a, E, N, P, C, D>( @@ -121,87 +142,89 @@ where // Get the nullifiers for the notes we are tracking let mut nullifiers = data_guard.get_nullifiers()?; - cache.with_blocks( - last_height, - limit, - Box::new(|block: CompactBlock| { - let current_height = block.height(); - - // Scanned blocks MUST be height-sequential. - if current_height != (last_height + 1) { - return Err(ChainInvalid::block_height_discontinuity( - last_height + 1, - current_height, - ) - .into()); - } - - let block_hash = BlockHash::from_slice(&block.hash); - let block_time = block.time; - - let txs: Vec> = { - let mut witness_refs: Vec<_> = witnesses.iter_mut().map(|w| &mut w.1).collect(); - - scan_block( - params, - block, - &extfvks, - &nullifiers, - &mut tree, - &mut witness_refs[..], - ) - }; - - // Enforce that all roots match. This is slow, so only include in debug builds. - #[cfg(debug_assertions)] - { - let cur_root = tree.root(); - for row in &witnesses { - if row.1.root() != cur_root { - return Err(Error::InvalidWitnessAnchor(row.0, current_height).into()); - } + cache + .with_blocks( + last_height, + limit, + Box::new(|block: CompactBlock| { + let current_height = block.height(); + + // Scanned blocks MUST be height-sequential. + if current_height != (last_height + 1) { + return Err(ChainInvalid::block_height_discontinuity( + last_height + 1, + current_height, + ) + .into()); } - for tx in &txs { - for output in tx.shielded_outputs.iter() { - if output.witness.root() != cur_root { - return Err(Error::InvalidNewWitnessAnchor( - output.index, - tx.txid, - current_height, - output.witness.root(), - ) - .into()); + + let block_hash = BlockHash::from_slice(&block.hash); + let block_time = block.time; + + let txs: Vec> = { + let mut witness_refs: Vec<_> = witnesses.iter_mut().map(|w| &mut w.1).collect(); + + scan_block( + params, + block, + &extfvks, + &nullifiers, + &mut tree, + &mut witness_refs[..], + ) + }; + + // Enforce that all roots match. This is slow, so only include in debug builds. + #[cfg(debug_assertions)] + { + let cur_root = tree.root(); + for row in &witnesses { + if row.1.root() != cur_root { + return Err(Error::InvalidWitnessAnchor(row.0, current_height).into()); + } + } + for tx in &txs { + for output in tx.shielded_outputs.iter() { + if output.witness.root() != cur_root { + return Err(Error::InvalidNewWitnessAnchor( + output.index, + tx.txid, + current_height, + output.witness.root(), + ) + .into()); + } } } } - } - - let new_witnesses = data_guard.advance_by_block( - &(PrunedBlock { - block_height: current_height, - block_hash, - block_time, - commitment_tree: &tree, - transactions: &txs, - }), - &witnesses, - )?; - - let spent_nf: Vec = txs - .iter() - .flat_map(|tx| tx.shielded_spends.iter().map(|spend| spend.nf)) - .collect(); - nullifiers.retain(|(_, nf)| !spent_nf.contains(nf)); - nullifiers.extend( - txs.iter() - .flat_map(|tx| tx.shielded_outputs.iter().map(|out| (out.account, out.nf))), - ); - - witnesses.extend(new_witnesses); - - last_height = current_height; - - Ok(()) - }), - ) + + let new_witnesses = data_guard.advance_by_block( + &(PrunedBlock { + block_height: current_height, + block_hash, + block_time, + commitment_tree: &tree, + transactions: &txs, + }), + &witnesses, + )?; + + let spent_nf: Vec = txs + .iter() + .flat_map(|tx| tx.shielded_spends.iter().map(|spend| spend.nf)) + .collect(); + nullifiers.retain(|(_, nf)| !spent_nf.contains(nf)); + nullifiers + .extend(txs.iter().flat_map(|tx| { + tx.shielded_outputs.iter().map(|out| (out.account, out.nf)) + })); + + witnesses.extend(new_witnesses); + + last_height = current_height; + + Ok(()) + }), + ) + .await } From c3ab476da0357b2322d69673360b003053780126 Mon Sep 17 00:00:00 2001 From: borngraced Date: Tue, 22 Aug 2023 14:03:15 +0100 Subject: [PATCH 04/42] use aync-ware mutex --- zcash_client_backend/Cargo.toml | 1 + .../src/with_async/data_api.rs | 30 ++++--------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index ba7f0af7a2..8eb7aab3ff 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -19,6 +19,7 @@ bls12_381 = "0.3.1" bs58 = { version = "0.4", features = ["check"] } base64 = "0.13" ff = "0.8" +futures = { version = "0.3", package = "futures", features = ["compat", "async-await"] } group = "0.8" hex = "0.4" jubjub = "0.5.1" diff --git a/zcash_client_backend/src/with_async/data_api.rs b/zcash_client_backend/src/with_async/data_api.rs index 9ffaa04054..0e3882b299 100644 --- a/zcash_client_backend/src/with_async/data_api.rs +++ b/zcash_client_backend/src/with_async/data_api.rs @@ -1,11 +1,12 @@ -use crate::proto::compact_formats::CompactBlock; -use std::fmt::Debug; -use std::sync::{Arc, Mutex}; - use crate::data_api::error::{ChainInvalid, Error}; use crate::data_api::{PrunedBlock, WalletWrite}; +use crate::proto::compact_formats::CompactBlock; use crate::wallet::{AccountId, WalletTx}; use crate::welding_rig::scan_block; + +use futures::lock::Mutex; +use std::fmt::Debug; +use std::sync::Arc; use zcash_primitives::block::BlockHash; use zcash_primitives::consensus; use zcash_primitives::consensus::{BlockHeight, NetworkUpgrade}; @@ -31,25 +32,6 @@ pub trait BlockSource { F: FnMut(CompactBlock) -> Result<(), Self::Error>; } -struct BlockSourceCaged; - -#[async_trait::async_trait(?Send)] -impl BlockSource for BlockSourceCaged { - type Error = String; - - async fn with_blocks( - &self, - from_height: BlockHeight, - limit: Option, - with_row: Box, - ) -> Result<(), Self::Error> - where - F: FnMut(CompactBlock) -> Result<(), Self::Error>, - { - todo!() - } -} - pub async fn validate_chain<'a, N, E, P, C>( parameters: &P, cache: &C, @@ -115,7 +97,7 @@ where N: Copy + Debug + Send, E: From> + Send + 'a, { - let mut data_guard = data.lock().unwrap(); + let mut data_guard = data.lock().await; let sapling_activation_height = params .activation_height(NetworkUpgrade::Sapling) .ok_or(Error::SaplingNotActive)?; From 924c418d7f0347dbb4d6497ecaf9a4e66fdaa6da Mon Sep 17 00:00:00 2001 From: borngraced Date: Fri, 25 Aug 2023 14:51:00 +0100 Subject: [PATCH 05/42] =?UTF-8?q?save=20dev=20state=20=E2=80=94=20ported?= =?UTF-8?q?=20wallet=5FWrite=20and=20Read?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- zcash_client_sqlite/Cargo.toml | 4 +- zcash_client_sqlite/src/with_async/mod.rs | 579 ++++++++++++++++++ .../src/with_async/wallet_actions.rs | 345 +++++++++++ 3 files changed, 927 insertions(+), 1 deletion(-) create mode 100644 zcash_client_sqlite/src/with_async/wallet_actions.rs diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 3a0f85a7bc..c6b1957d06 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -13,14 +13,16 @@ license = "MIT OR Apache-2.0" edition = "2018" [dependencies] +async-trait = "0.1.52" bech32 = "0.9.1" bs58 = { version = "0.4", features = ["check"] } ff = "0.8" +futures = { version = "0.3", package = "futures", features = ["compat", "async-await"] } group = "0.8" jubjub = "0.5.1" protobuf = "2.20" rand_core = "0.5.1" -rusqlite = { version = "0.28", features = ["time"] } +rusqlite = { version = "0.28", features = ["bundled", "time"] } libsqlite3-sys= { version = "0.25.2", features = ["bundled"] } time = "0.3" zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } diff --git a/zcash_client_sqlite/src/with_async/mod.rs b/zcash_client_sqlite/src/with_async/mod.rs index e69de29bb2..4e54601fc2 100644 --- a/zcash_client_sqlite/src/with_async/mod.rs +++ b/zcash_client_sqlite/src/with_async/mod.rs @@ -0,0 +1,579 @@ +pub mod wallet_actions; + +use std::cmp; +use std::collections::HashMap; +use std::fmt::Debug; +use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET; +use zcash_client_backend::data_api::{PrunedBlock, ReceivedTransaction, SentTransaction}; +use zcash_client_backend::wallet::{AccountId, SpendableNote}; +use zcash_primitives::block::BlockHash; +use zcash_primitives::consensus::BlockHeight; +use zcash_primitives::memo::Memo; +use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness}; +use zcash_primitives::sapling::{Node, Nullifier, PaymentAddress}; +use zcash_primitives::transaction::components::Amount; +use zcash_primitives::transaction::TxId; +use zcash_primitives::zip32::ExtendedFullViewingKey; + +#[async_trait::async_trait] +pub trait WalletRead: Send + Sync + 'static { + type Error; + type NoteRef: Copy + Debug; + type TxRef: Copy + Debug; + + /// Returns the minimum and maximum block heights for stored blocks. + /// + /// This will return `Ok(None)` if no block data is present in the database. + async fn block_height_extrema(&self) + -> Result, Self::Error>; + + /// Returns the default target height (for the block in which a new + /// transaction would be mined) and anchor height (to use for a new + /// transaction), given the range of block heights that the backend + /// knows about. + /// + /// This will return `Ok(None)` if no block data is present in the database. + async fn get_target_and_anchor_heights( + &self, + ) -> Result, Self::Error> { + self.block_height_extrema().await.map(|heights| { + heights.map(|(min_height, max_height)| { + let target_height = max_height + 1; + + // Select an anchor ANCHOR_OFFSET back from the target block, + // unless that would be before the earliest block we have. + let anchor_height = BlockHeight::from(cmp::max( + u32::from(target_height).saturating_sub(ANCHOR_OFFSET), + u32::from(min_height), + )); + + (target_height, anchor_height) + }) + }) + } + + /// Returns the block hash for the block at the given height, if the + /// associated block data is available. Returns `Ok(None)` if the hash + /// is not found in the database. + async fn get_block_hash( + &self, + block_height: BlockHeight, + ) -> Result, Self::Error>; + + /// Returns the block hash for the block at the maximum height known + /// in stored data. + /// + /// This will return `Ok(None)` if no block data is present in the database. + async fn get_max_height_hash(&self) -> Result, Self::Error> { + let extrema = self.block_height_extrema().await?; + let res = if let Some((_, max_height)) = extrema { + self.get_block_hash(max_height) + .await + .map(|hash_opt| hash_opt.map(move |hash| (max_height, hash)))? + } else { + None + }; + + Ok(res) + } + + /// Returns the block height in which the specified transaction was mined, + /// or `Ok(None)` if the transaction is not mined in the main chain. + async fn get_tx_height(&self, txid: TxId) -> Result, Self::Error>; + + /// Returns the payment address for the specified account, if the account + /// identifier specified refers to a valid account for this wallet. + /// + /// This will return `Ok(None)` if the account identifier does not correspond + /// to a known account. + async fn get_address(&self, account: AccountId) -> Result, Self::Error>; + + /// Returns all extended full viewing keys known about by this wallet. + async fn get_extended_full_viewing_keys( + &self, + ) -> Result, Self::Error>; + + /// Checks whether the specified extended full viewing key is + /// associated with the account. + async fn is_valid_account_extfvk( + &self, + account: AccountId, + extfvk: &ExtendedFullViewingKey, + ) -> Result; + + /// Returns the wallet balance for an account as of the specified block + /// height. + /// + /// This may be used to obtain a balance that ignores notes that have been + /// received so recently that they are not yet deemed spendable. + async fn get_balance_at( + &self, + account: AccountId, + anchor_height: BlockHeight, + ) -> Result; + + /// Returns the memo for a note. + /// + /// Implementations of this method must return an error if the note identifier + /// does not appear in the backing data store. + async fn get_memo(&self, id_note: Self::NoteRef) -> Result; + + /// Returns the note commitment tree at the specified block height. + async fn get_commitment_tree( + &self, + block_height: BlockHeight, + ) -> Result>, Self::Error>; + + /// Returns the incremental witnesses as of the specified block height. + #[allow(clippy::type_complexity)] + async fn get_witnesses( + &self, + block_height: BlockHeight, + ) -> Result)>, Self::Error>; + + /// Returns the unspent nullifiers, along with the account identifiers + /// with which they are associated. + async fn get_nullifiers(&self) -> Result, Self::Error>; + + /// Return all spendable notes. + async fn get_spendable_notes( + &self, + account: AccountId, + anchor_height: BlockHeight, + ) -> Result, Self::Error>; + + /// Returns a list of spendable notes sufficient to cover the specified + /// target value, if possible. + async fn select_spendable_notes( + &self, + account: AccountId, + target_value: Amount, + anchor_height: BlockHeight, + ) -> Result, Self::Error>; +} + +/// This trait encapsulates the write capabilities required to update stored +/// wallet data. +#[async_trait::async_trait] +pub trait WalletWrite: WalletRead { + #[allow(clippy::type_complexity)] + async fn advance_by_block( + &mut self, + block: &PrunedBlock, + updated_witnesses: &[(Self::NoteRef, IncrementalWitness)], + ) -> Result)>, Self::Error>; + + async fn store_received_tx( + &mut self, + received_tx: &ReceivedTransaction, + ) -> Result; + + async fn store_sent_tx( + &mut self, + sent_tx: &SentTransaction, + ) -> Result; + + /// Rewinds the wallet database to the specified height. + /// + /// This method assumes that the state of the underlying data store is + /// consistent up to a particular block height. Since it is possible that + /// a chain reorg might invalidate some stored state, this method must be + /// implemented in order to allow users of this API to "reset" the data store + /// to correctly represent chainstate as of a specified block height. + /// + /// After calling this method, the block at the given height will be the + /// most recent block and all other operations will treat this block + /// as the chain tip for balance determination purposes. + /// + /// There may be restrictions on how far it is possible to rewind. + async fn rewind_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error>; +} + +use crate::error::SqliteClientError; +use crate::{wallet, NoteId, WalletDb}; +use rusqlite::{Connection, OptionalExtension, Statement, ToSql}; +use std::sync::{Arc, Mutex}; +use zcash_client_backend::encoding::{ + decode_extended_full_viewing_key, decode_payment_address, encode_extended_full_viewing_key, +}; +use zcash_primitives::consensus; + +/// A wrapper for the SQLite connection to the wallet database. +#[derive(Clone)] +pub struct WalletDbAsync

{ + inner: Arc>>, +} + +impl WalletDbAsync

{ + /// Given a wallet database connection, obtain a handle for the write operations + /// for that database. This operation may eagerly initialize and cache sqlite + /// prepared statements that are used in write operations. + pub fn get_update_ops(&self) -> Result, SqliteClientError> { + Ok(DataConnStmtCacheAsync { + wallet_db: self.clone(), + }) + } +} + +#[async_trait::async_trait] +impl WalletRead for WalletDbAsync

{ + type Error = SqliteClientError; + type NoteRef = NoteId; + type TxRef = i64; + + async fn block_height_extrema( + &self, + ) -> Result, Self::Error> { + let db = self.inner.lock().unwrap(); + wallet::block_height_extrema(&db).map_err(SqliteClientError::from) + } + + async fn get_block_hash( + &self, + block_height: BlockHeight, + ) -> Result, Self::Error> { + let db = self.inner.lock().unwrap(); + wallet::get_block_hash(&db, block_height).map_err(SqliteClientError::from) + } + + async fn get_tx_height(&self, txid: TxId) -> Result, Self::Error> { + let db = self.inner.lock().unwrap(); + wallet::get_tx_height(&db, txid).map_err(SqliteClientError::from) + } + + async fn get_address(&self, account: AccountId) -> Result, Self::Error> { + let db = self.inner.lock().unwrap(); + wallet::get_address(&db, account).map_err(SqliteClientError::from) + } + + async fn get_extended_full_viewing_keys( + &self, + ) -> Result, Self::Error> { + let db = self.inner.lock().unwrap(); + wallet::get_extended_full_viewing_keys(&db) + } + + async fn is_valid_account_extfvk( + &self, + account: AccountId, + extfvk: &ExtendedFullViewingKey, + ) -> Result { + let db = self.inner.lock().unwrap(); + wallet::is_valid_account_extfvk(&db, account, extfvk) + } + + async fn get_balance_at( + &self, + account: AccountId, + anchor_height: BlockHeight, + ) -> Result { + let db = self.inner.lock().unwrap(); + wallet::get_balance_at(&db, account, anchor_height) + } + + async fn get_memo(&self, id_note: Self::NoteRef) -> Result { + let db = self.inner.lock().unwrap(); + match id_note { + NoteId::SentNoteId(id_note) => wallet::get_sent_memo(&db, id_note), + NoteId::ReceivedNoteId(id_note) => wallet::get_received_memo(&db, id_note), + } + } + + async fn get_commitment_tree( + &self, + block_height: BlockHeight, + ) -> Result>, Self::Error> { + let db = self.inner.lock().unwrap(); + wallet::get_commitment_tree(&db, block_height) + } + + #[allow(clippy::type_complexity)] + async fn get_witnesses( + &self, + block_height: BlockHeight, + ) -> Result)>, Self::Error> { + let db = self.inner.lock().unwrap(); + wallet::get_witnesses(&db, block_height) + } + + async fn get_nullifiers(&self) -> Result, Self::Error> { + let db = self.inner.lock().unwrap(); + wallet::get_nullifiers(&db) + } + + async fn get_spendable_notes( + &self, + account: AccountId, + anchor_height: BlockHeight, + ) -> Result, Self::Error> { + let db = self.inner.lock().unwrap(); + wallet::transact::get_spendable_notes(&db, account, anchor_height) + } + + async fn select_spendable_notes( + &self, + account: AccountId, + target_value: Amount, + anchor_height: BlockHeight, + ) -> Result, Self::Error> { + let db = self.inner.lock().unwrap(); + wallet::transact::select_spendable_notes(&db, account, target_value, anchor_height) + } +} + +pub struct DataConnStmtCacheAsync

{ + wallet_db: WalletDbAsync

, +} + +impl DataConnStmtCacheAsync

{ + fn transactionally(&mut self, f: F) -> Result + where + F: FnOnce(&mut Self) -> Result, + { + self.wallet_db + .inner + .lock() + .unwrap() + .conn + .execute("BEGIN IMMEDIATE", [])?; + match f(self) { + Ok(result) => { + self.wallet_db + .inner + .lock() + .unwrap() + .conn + .execute("COMMIT", [])?; + Ok(result) + } + Err(error) => { + match self.wallet_db.inner.lock().unwrap().conn.execute("ROLLBACK", []) { + Ok(_) => Err(error), + Err(e) => + // Panicking here is probably the right thing to do, because it + // means the database is corrupt. + panic!( + "Rollback failed with error {} while attempting to recover from error {}; database is likely corrupt.", + e, + error + ) + } + } + } + } +} + +#[async_trait::async_trait] +impl WalletRead for DataConnStmtCacheAsync

{ + type Error = SqliteClientError; + type NoteRef = NoteId; + type TxRef = i64; + + async fn block_height_extrema( + &self, + ) -> Result, Self::Error> { + self.block_height_extrema().await + } + + async fn get_block_hash( + &self, + block_height: BlockHeight, + ) -> Result, Self::Error> { + self.get_block_hash(block_height).await + } + + async fn get_tx_height(&self, txid: TxId) -> Result, Self::Error> { + self.get_tx_height(txid).await + } + + async fn get_address(&self, account: AccountId) -> Result, Self::Error> { + self.get_address(account).await + } + + async fn get_extended_full_viewing_keys( + &self, + ) -> Result, Self::Error> { + self.get_extended_full_viewing_keys().await + } + + async fn is_valid_account_extfvk( + &self, + account: AccountId, + extfvk: &ExtendedFullViewingKey, + ) -> Result { + self.is_valid_account_extfvk(account, extfvk).await + } + + async fn get_balance_at( + &self, + account: AccountId, + anchor_height: BlockHeight, + ) -> Result { + self.get_balance_at(account, anchor_height).await + } + + async fn get_memo(&self, id_note: Self::NoteRef) -> Result { + self.get_memo(id_note).await + } + + async fn get_commitment_tree( + &self, + block_height: BlockHeight, + ) -> Result>, Self::Error> { + self.get_commitment_tree(block_height).await + } + + #[allow(clippy::type_complexity)] + async fn get_witnesses( + &self, + block_height: BlockHeight, + ) -> Result)>, Self::Error> { + self.get_witnesses(block_height).await + } + + async fn get_nullifiers(&self) -> Result, Self::Error> { + self.get_nullifiers().await + } + + async fn get_spendable_notes( + &self, + account: AccountId, + anchor_height: BlockHeight, + ) -> Result, Self::Error> { + self.get_spendable_notes(account, anchor_height).await + } + + async fn select_spendable_notes( + &self, + account: AccountId, + target_value: Amount, + anchor_height: BlockHeight, + ) -> Result, Self::Error> { + self.select_spendable_notes(account, target_value, anchor_height) + .await + } +} + +#[async_trait::async_trait] +impl WalletWrite for DataConnStmtCacheAsync

{ + #[allow(clippy::type_complexity)] + async fn advance_by_block( + &mut self, + block: &PrunedBlock, + updated_witnesses: &[(Self::NoteRef, IncrementalWitness)], + ) -> Result)>, Self::Error> { + // database updates for each block are transactional + self.transactionally(|up| { + let db = up.wallet_db.inner.lock().unwrap(); + // Insert the block into the database. + wallet_actions::insert_block( + &db, + block.block_height, + block.block_hash, + block.block_time, + &block.commitment_tree, + )?; + + let mut new_witnesses = vec![]; + for tx in block.transactions { + let tx_row = wallet_actions::put_tx_meta(&db, &tx, block.block_height)?; + + // Mark notes as spent and remove them from the scanning cache + for spend in &tx.shielded_spends { + wallet_actions::mark_spent(&db, tx_row, &spend.nf)?; + } + + for output in &tx.shielded_outputs { + let received_note_id = wallet_actions::put_received_note(&db, output, tx_row)?; + + // Save witness for note. + new_witnesses.push((received_note_id, output.witness.clone())); + } + } + + // Insert current new_witnesses into the database. + for (received_note_id, witness) in updated_witnesses.iter().chain(new_witnesses.iter()) + { + if let NoteId::ReceivedNoteId(rnid) = *received_note_id { + wallet_actions::insert_witness(&db, rnid, witness, block.block_height)?; + } else { + return Err(SqliteClientError::InvalidNoteId); + } + } + + // Prune the stored witnesses (we only expect rollbacks of at most 100 blocks). + let below_height = if block.block_height < BlockHeight::from(100) { + BlockHeight::from(0) + } else { + block.block_height - 100 + }; + wallet_actions::prune_witnesses(&db, below_height)?; + + // Update now-expired transactions that didn't get mined. + wallet_actions::update_expired_notes(&db, block.block_height)?; + + Ok(new_witnesses) + }) + } + + async fn store_received_tx( + &mut self, + received_tx: &ReceivedTransaction, + ) -> Result { + self.transactionally(|up| { + let db = up.wallet_db.inner.lock().unwrap(); + let tx_ref = wallet_actions::put_tx_data(&db, received_tx.tx, None)?; + + for output in received_tx.outputs { + if output.outgoing { + wallet_actions::put_sent_note(&db, output, tx_ref)?; + } else { + wallet_actions::put_received_note(&db, output, tx_ref)?; + } + } + + Ok(tx_ref) + }) + } + + async fn store_sent_tx( + &mut self, + sent_tx: &SentTransaction, + ) -> Result { + // Update the database atomically, to ensure the result is internally consistent. + self.transactionally(|up| { + let db = up.wallet_db.inner.lock().unwrap(); + let tx_ref = wallet_actions::put_tx_data(&db, &sent_tx.tx, Some(sent_tx.created))?; + + // Mark notes as spent. + // + // This locks the notes so they aren't selected again by a subsequent call to + // create_spend_to_address() before this transaction has been mined (at which point the notes + // get re-marked as spent). + // + // Assumes that create_spend_to_address() will never be called in parallel, which is a + // reasonable assumption for a light client such as a mobile phone. + for spend in &sent_tx.tx.shielded_spends { + wallet_actions::mark_spent(&db, tx_ref, &spend.nullifier)?; + } + + wallet_actions::insert_sent_note( + &db, + tx_ref, + sent_tx.output_index, + sent_tx.account, + sent_tx.recipient_address, + sent_tx.value, + sent_tx.memo.as_ref(), + )?; + + // Return the row number of the transaction, so the caller can fetch it for sending. + Ok(tx_ref) + }) + } + + async fn rewind_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error> { + let db = self.wallet_db.inner.lock().unwrap(); + wallet::rewind_to_height(&db, block_height) + } +} diff --git a/zcash_client_sqlite/src/with_async/wallet_actions.rs b/zcash_client_sqlite/src/with_async/wallet_actions.rs new file mode 100644 index 0000000000..6465702319 --- /dev/null +++ b/zcash_client_sqlite/src/with_async/wallet_actions.rs @@ -0,0 +1,345 @@ +use crate::error::SqliteClientError; +use crate::wallet::ShieldedOutput; +use crate::with_async::WalletDbAsync; +use crate::{NoteId, WalletDb}; +use ff::PrimeField; +use rusqlite::{params, ToSql}; +use std::sync::MutexGuard; +use zcash_client_backend::address::RecipientAddress; +use zcash_client_backend::encoding::encode_payment_address; +use zcash_client_backend::wallet::{AccountId, WalletTx}; +use zcash_client_backend::DecryptedOutput; +use zcash_primitives::block::BlockHash; +use zcash_primitives::consensus; +use zcash_primitives::consensus::BlockHeight; +use zcash_primitives::memo::MemoBytes; +use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness}; +use zcash_primitives::sapling::{Node, Nullifier}; +use zcash_primitives::transaction::components::Amount; +use zcash_primitives::transaction::Transaction; + +/// Inserts information about a scanned block into the database. +pub fn insert_block

( + db: &MutexGuard>, + block_height: BlockHeight, + block_hash: BlockHash, + block_time: u32, + commitment_tree: &CommitmentTree, +) -> Result<(), SqliteClientError> { + let mut encoded_tree = Vec::new(); + commitment_tree.write(&mut encoded_tree).unwrap(); + + db.conn + .prepare( + "INSERT INTO blocks (height, hash, time, sapling_tree) + VALUES (?, ?, ?, ?)", + )? + .execute(params![ + u32::from(block_height), + &block_hash.0[..], + block_time, + encoded_tree + ])?; + + Ok(()) +} + +/// Inserts information about a mined transaction that was observed to +/// contain a note related to this wallet into the database. +pub fn put_tx_meta( + db: &MutexGuard>, + tx: &WalletTx, + height: BlockHeight, +) -> Result { + let txid = tx.txid.0.to_vec(); + if db + .conn + .prepare( + "UPDATE transactions + SET block = ?, tx_index = ? WHERE txid = ?", + )? + .execute(params![u32::from(height), (tx.index as i64), txid])? + == 0 + { + // It isn't there, so insert our transaction into the database. + db.conn + .prepare( + "INSERT INTO transactions (txid, block, tx_index) + VALUES (?, ?, ?)", + )? + .execute(params![txid, u32::from(height), (tx.index as i64),])?; + + Ok(db.conn.last_insert_rowid()) + } else { + // It was there, so grab its row number. + db.conn + .prepare("SELECT id_tx FROM transactions WHERE txid = ?")? + .query_row([txid], |row| row.get(0)) + .map_err(SqliteClientError::from) + } +} + +/// Marks a given nullifier as having been revealed in the construction +/// of the specified transaction. +/// +/// Marking a note spent in this fashion does NOT imply that the +/// spending transaction has been mined. +pub fn mark_spent<'a, P>( + db: &MutexGuard>, + tx_ref: i64, + nf: &Nullifier, +) -> Result<(), SqliteClientError> { + db.conn + .prepare("UPDATE received_notes SET spent = ? WHERE nf = ?")? + .execute([tx_ref.to_sql()?, nf.0.to_sql()?])?; + Ok(()) +} + +/// Records the specified shielded output as having been received. +// Assumptions: +// - A transaction will not contain more than 2^63 shielded outputs. +// - A note value will never exceed 2^63 zatoshis. +pub fn put_received_note( + db: &MutexGuard>, + output: &T, + tx_ref: i64, +) -> Result { + let rcm = output.note().rcm().to_repr(); + let account = output.account().0 as i64; + let diversifier = output.to().diversifier().0.to_vec(); + let value = output.note().value as i64; + let rcm = rcm.as_ref(); + let memo = output.memo().map(|m| m.as_slice()); + let is_change = output.is_change(); + let tx = tx_ref; + let output_index = output.index() as i64; + let nf_bytes = output.nullifier().map(|nf| nf.0.to_vec()); + + let sql_args: &[(&str, &dyn ToSql)] = &[ + (&":account", &account), + (&":diversifier", &diversifier), + (&":value", &value), + (&":rcm", &rcm), + (&":nf", &nf_bytes), + (&":memo", &memo), + (&":is_change", &is_change), + (&":tx", &tx), + (&":output_index", &output_index), + ]; + + // First try updating an existing received note into the database. + if db + .conn + .prepare( + "UPDATE received_notes + SET account = :account, + diversifier = :diversifier, + value = :value, + rcm = :rcm, + nf = IFNULL(:nf, nf), + memo = IFNULL(:memo, memo), + is_change = IFNULL(:is_change, is_change) + WHERE tx = :tx AND output_index = :output_index", + )? + .execute_named(&sql_args)? + == 0 + { + // It isn't there, so insert our note into the database. + db.conn + .prepare( + "UPDATE received_notes + SET account = :account, + diversifier = :diversifier, + value = :value, + rcm = :rcm, + nf = IFNULL(:nf, nf), + memo = IFNULL(:memo, memo), + is_change = IFNULL(:is_change, is_change) + WHERE tx = :tx AND output_index = :output_index", + )? + .execute_named(&sql_args)?; + + Ok(NoteId::ReceivedNoteId(db.conn.last_insert_rowid())) + } else { + // It was there, so grab its row number. + db.conn + .prepare("SELECT id_note FROM received_notes WHERE tx = ? AND output_index = ?")? + .query_row(params![tx_ref, (output.index() as i64)], |row| { + row.get(0).map(NoteId::ReceivedNoteId) + }) + .map_err(SqliteClientError::from) + } +} + +/// Records the incremental witness for the specified note, +/// as of the given block height. +pub fn insert_witness

( + db: &MutexGuard>, + note_id: i64, + witness: &IncrementalWitness, + height: BlockHeight, +) -> Result<(), SqliteClientError> { + let mut encoded = Vec::new(); + witness.write(&mut encoded).unwrap(); + + db.conn + .prepare( + "INSERT INTO sapling_witnesses (note, block, witness) + VALUES (?, ?, ?)", + )? + .execute(params![note_id, u32::from(height), encoded])?; + + Ok(()) +} + +/// Removes old incremental witnesses up to the given block height. +pub fn prune_witnesses

( + db: &MutexGuard>, + below_height: BlockHeight, +) -> Result<(), SqliteClientError> { + db.conn + .prepare("DELETE FROM sapling_witnesses WHERE block < ?")? + .execute([u32::from(below_height)])?; + Ok(()) +} + +/// Marks notes that have not been mined in transactions +/// as expired, up to the given block height. +pub fn update_expired_notes

( + db: &MutexGuard>, + height: BlockHeight, +) -> Result<(), SqliteClientError> { + db.conn + .prepare( + "UPDATE received_notes SET spent = NULL WHERE EXISTS ( + SELECT id_tx FROM transactions + WHERE id_tx = received_notes.spent AND block IS NULL AND expiry_height < ? + )", + )? + .execute([u32::from(height)])?; + Ok(()) +} + +/// Inserts full transaction data into the database. +pub fn put_tx_data

( + db: &MutexGuard>, + tx: &Transaction, + created_at: Option, +) -> Result { + let txid = tx.txid().0.to_vec(); + + let mut raw_tx = vec![]; + tx.write(&mut raw_tx)?; + + if db + .conn + .prepare( + "UPDATE transactions + SET expiry_height = ?, raw = ? WHERE txid = ?", + )? + .execute(params![u32::from(tx.expiry_height), raw_tx, txid,])? + == 0 + { + // It isn't there, so insert our transaction into the database. + db.conn + .prepare( + "INSERT INTO transactions (txid, created, expiry_height, raw) + VALUES (?, ?, ?, ?)", + )? + .execute(params![ + txid, + created_at, + u32::from(tx.expiry_height), + raw_tx + ])?; + + Ok(db.conn.last_insert_rowid()) + } else { + // It was there, so grab its row number. + db.conn + .prepare("SELECT id_tx FROM transactions WHERE txid = ?")? + .query_row([txid], |row| row.get(0)) + .map_err(SqliteClientError::from) + } +} + +/// Records information about a note that your wallet created. +pub fn put_sent_note( + db: &MutexGuard>, + output: &DecryptedOutput, + tx_ref: i64, +) -> Result<(), SqliteClientError> { + let output_index = output.index as i64; + let account = output.account.0 as i64; + let value = output.note.value as i64; + let to_str = encode_payment_address(db.params.hrp_sapling_payment_address(), &output.to); + + // Try updating an existing sent note. + if db + .conn + .prepare( + "UPDATE sent_notes + SET from_account = ?, address = ?, value = ?, memo = ? + WHERE tx = ? AND output_index = ?", + )? + .execute(params![ + account, + to_str, + value, + &output.memo.as_slice(), + tx_ref, + output_index + ])? + == 0 + { + // It isn't there, so insert. + insert_sent_note( + db, + tx_ref, + output.index, + output.account, + &RecipientAddress::Shielded(output.to.clone()), + Amount::from_u64(output.note.value) + .map_err(|_| SqliteClientError::CorruptedData("Note value invalid.".to_string()))?, + Some(&output.memo), + )? + } + + Ok(()) +} + +/// Inserts a sent note into the wallet database. +/// +/// `output_index` is the index within the transaction that contains the recipient output: +/// +/// - If `to` is a Sapling address, this is an index into the Sapling outputs of the +/// transaction. +/// - If `to` is a transparent address, this is an index into the transparent outputs of +/// the transaction. +pub fn insert_sent_note( + db: &MutexGuard>, + tx_ref: i64, + output_index: usize, + account: AccountId, + to: &RecipientAddress, + value: Amount, + memo: Option<&MemoBytes>, +) -> Result<(), SqliteClientError> { + let to_str = to.encode(&db.params); + let ivalue: i64 = value.into(); + db.conn + .prepare( + "INSERT INTO sent_notes (tx, output_index, from_account, address, value, memo) + VALUES (?, ?, ?, ?, ?, ?)", + )? + .execute(params![ + tx_ref, + (output_index as i64), + account.0, + to_str, + ivalue, + memo.map(|m| m.as_slice().to_vec()), + ])?; + + Ok(()) +} From 3b1122b16aaa74d1fd6543d09a0e28ea77d7bdd3 Mon Sep 17 00:00:00 2001 From: borngraced Date: Fri, 25 Aug 2023 15:04:59 +0100 Subject: [PATCH 06/42] =?UTF-8?q?save=20dev=20state=20=E2=80=94=20impl=20i?= =?UTF-8?q?nner=20fn=20for=20WalletDbAsync?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- zcash_client_sqlite/src/with_async/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zcash_client_sqlite/src/with_async/mod.rs b/zcash_client_sqlite/src/with_async/mod.rs index 4e54601fc2..803de1bfda 100644 --- a/zcash_client_sqlite/src/with_async/mod.rs +++ b/zcash_client_sqlite/src/with_async/mod.rs @@ -205,6 +205,9 @@ pub struct WalletDbAsync

{ } impl WalletDbAsync

{ + pub fn inner(&self) -> Arc>> { + self.inner.clone() + } /// Given a wallet database connection, obtain a handle for the write operations /// for that database. This operation may eagerly initialize and cache sqlite /// prepared statements that are used in write operations. From 34799ce21a66feb95e36168ef5bf1da1fb4ee307 Mon Sep 17 00:00:00 2001 From: borngraced Date: Fri, 25 Aug 2023 19:41:20 +0100 Subject: [PATCH 07/42] =?UTF-8?q?save=20dev=20state=20=E2=80=94=20impl=20f?= =?UTF-8?q?or=5Fpath=20fn=20for=20WalletDbAsync?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- zcash_client_sqlite/src/with_async/mod.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zcash_client_sqlite/src/with_async/mod.rs b/zcash_client_sqlite/src/with_async/mod.rs index 803de1bfda..0c25989024 100644 --- a/zcash_client_sqlite/src/with_async/mod.rs +++ b/zcash_client_sqlite/src/with_async/mod.rs @@ -3,6 +3,7 @@ pub mod wallet_actions; use std::cmp; use std::collections::HashMap; use std::fmt::Debug; +use std::path::Path; use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET; use zcash_client_backend::data_api::{PrunedBlock, ReceivedTransaction, SentTransaction}; use zcash_client_backend::wallet::{AccountId, SpendableNote}; @@ -208,6 +209,15 @@ impl WalletDbAsync

{ pub fn inner(&self) -> Arc>> { self.inner.clone() } + + /// Construct a connection to the wallet database stored at the specified path. + pub fn for_path>(path: F, params: P) -> Result { + let db = Connection::open(path).map(move |conn| WalletDb { conn, params })?; + Ok(Self { + inner: Arc::new(Mutex::new(db)), + }) + } + /// Given a wallet database connection, obtain a handle for the write operations /// for that database. This operation may eagerly initialize and cache sqlite /// prepared statements that are used in write operations. From 9d2c681740f25c11984af9ac38c899a154d1618f Mon Sep 17 00:00:00 2001 From: borngraced Date: Fri, 25 Aug 2023 21:36:55 +0100 Subject: [PATCH 08/42] =?UTF-8?q?save=20dev=20state=20=E2=80=94=20make=20B?= =?UTF-8?q?lockSource=20Send?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- zcash_client_backend/src/with_async/data_api.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/src/with_async/data_api.rs b/zcash_client_backend/src/with_async/data_api.rs index 0e3882b299..4811ae4c5f 100644 --- a/zcash_client_backend/src/with_async/data_api.rs +++ b/zcash_client_backend/src/with_async/data_api.rs @@ -16,7 +16,7 @@ use zcash_primitives::zip32::ExtendedFullViewingKey; /// This trait provides sequential access to raw blockchain data via a callback-oriented /// API. -#[async_trait::async_trait(?Send)] +#[async_trait::async_trait] pub trait BlockSource { type Error; @@ -26,7 +26,7 @@ pub trait BlockSource { &self, from_height: BlockHeight, limit: Option, - with_row: Box, + with_row: F, ) -> Result<(), Self::Error> where F: FnMut(CompactBlock) -> Result<(), Self::Error>; From 329e2495345ab8fb5003729fd41b4b45448e133a Mon Sep 17 00:00:00 2001 From: borngraced Date: Sun, 27 Aug 2023 13:08:08 +0100 Subject: [PATCH 09/42] =?UTF-8?q?save=20dev=20state=20=E2=80=94=20finish?= =?UTF-8?q?=20modification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- zcash_client_backend/src/data_api.rs | 2 + zcash_client_backend/src/lib.rs | 1 - .../src/with_async/data_api.rs | 212 ------------------ zcash_client_backend/src/with_async/mod.rs | 1 - zcash_client_sqlite/Cargo.toml | 1 + zcash_client_sqlite/src/with_async/mod.rs | 183 ++++++++++----- .../src/with_async/wallet_actions.rs | 4 +- 7 files changed, 134 insertions(+), 270 deletions(-) delete mode 100644 zcash_client_backend/src/with_async/data_api.rs delete mode 100644 zcash_client_backend/src/with_async/mod.rs diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 166fd80cba..ff87bde87e 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -178,6 +178,7 @@ pub trait WalletRead { /// The subset of information that is relevant to this wallet that has been /// decrypted and extracted from a [CompactBlock]. +#[derive(Clone)] pub struct PrunedBlock<'a> { pub block_height: BlockHeight, pub block_hash: BlockHash, @@ -201,6 +202,7 @@ pub struct ReceivedTransaction<'a> { /// The purpose of this struct is to permit atomic updates of the /// wallet database when transactions are created and submitted /// to the network. +#[derive(Clone)] pub struct SentTransaction<'a> { pub tx: &'a Transaction, pub created: time::OffsetDateTime, diff --git a/zcash_client_backend/src/lib.rs b/zcash_client_backend/src/lib.rs index 011e329c0c..085070c134 100644 --- a/zcash_client_backend/src/lib.rs +++ b/zcash_client_backend/src/lib.rs @@ -16,7 +16,6 @@ pub mod keys; pub mod proto; pub mod wallet; pub mod welding_rig; -pub mod with_async; pub mod zip321; pub use decrypt::{decrypt_transaction, DecryptedOutput}; diff --git a/zcash_client_backend/src/with_async/data_api.rs b/zcash_client_backend/src/with_async/data_api.rs deleted file mode 100644 index 4811ae4c5f..0000000000 --- a/zcash_client_backend/src/with_async/data_api.rs +++ /dev/null @@ -1,212 +0,0 @@ -use crate::data_api::error::{ChainInvalid, Error}; -use crate::data_api::{PrunedBlock, WalletWrite}; -use crate::proto::compact_formats::CompactBlock; -use crate::wallet::{AccountId, WalletTx}; -use crate::welding_rig::scan_block; - -use futures::lock::Mutex; -use std::fmt::Debug; -use std::sync::Arc; -use zcash_primitives::block::BlockHash; -use zcash_primitives::consensus; -use zcash_primitives::consensus::{BlockHeight, NetworkUpgrade}; -use zcash_primitives::merkle_tree::CommitmentTree; -use zcash_primitives::sapling::Nullifier; -use zcash_primitives::zip32::ExtendedFullViewingKey; - -/// This trait provides sequential access to raw blockchain data via a callback-oriented -/// API. -#[async_trait::async_trait] -pub trait BlockSource { - type Error; - - /// Scan the specified `limit` number of blocks from the blockchain, starting at - /// `from_height`, applying the provided callback to each block. - async fn with_blocks( - &self, - from_height: BlockHeight, - limit: Option, - with_row: F, - ) -> Result<(), Self::Error> - where - F: FnMut(CompactBlock) -> Result<(), Self::Error>; -} - -pub async fn validate_chain<'a, N, E, P, C>( - parameters: &P, - cache: &C, - validate_from: Option<(BlockHeight, BlockHash)>, -) -> Result<(), E> -where - E: From> + Send + 'a, - P: consensus::Parameters, - C: BlockSource, -{ - let sapling_activation_height = parameters - .activation_height(NetworkUpgrade::Sapling) - .ok_or(Error::SaplingNotActive)?; - - // The cache will contain blocks above the `validate_from` height. Validate from that maximum - // height up to the chain tip, returning the hash of the block found in the cache at the - // `validate_from` height, which can then be used to verify chain integrity by comparing - // against the `validate_from` hash. - let from_height = validate_from - .map(|(height, _)| height) - .unwrap_or(sapling_activation_height - 1); - - let mut prev_height = from_height; - let mut prev_hash: Option = validate_from.map(|(_, hash)| hash); - - cache - .with_blocks( - from_height, - None, - Box::new(|block: CompactBlock| { - let current_height = block.height(); - let result = if current_height != prev_height + 1 { - Err(ChainInvalid::block_height_discontinuity( - prev_height + 1, - current_height, - )) - } else { - match prev_hash { - None => Ok(()), - Some(h) if h == block.prev_hash() => Ok(()), - Some(_) => Err(ChainInvalid::prev_hash_mismatch(current_height)), - } - }; - - prev_height = current_height; - prev_hash = Some(block.hash()); - result.map_err(E::from) - }), - ) - .await -} - -pub async fn scan_cached_blocks<'a, E, N, P, C, D>( - params: &P, - cache: &C, - data: Arc>, - limit: Option, -) -> Result<(), E> -where - P: consensus::Parameters + Send + Sync, - C: BlockSource, - D: WalletWrite, - N: Copy + Debug + Send, - E: From> + Send + 'a, -{ - let mut data_guard = data.lock().await; - let sapling_activation_height = params - .activation_height(NetworkUpgrade::Sapling) - .ok_or(Error::SaplingNotActive)?; - - // Recall where we synced up to previously. - // If we have never synced, use sapling activation height to select all cached CompactBlocks. - let mut last_height = data_guard.block_height_extrema().map(|opt| { - opt.map(|(_, max)| max) - .unwrap_or(sapling_activation_height - 1) - })?; - - // Fetch the ExtendedFullViewingKeys we are tracking - let extfvks = data_guard.get_extended_full_viewing_keys()?; - let extfvks: Vec<(&AccountId, &ExtendedFullViewingKey)> = extfvks.iter().collect(); - - // Get the most recent CommitmentTree - let mut tree = data_guard - .get_commitment_tree(last_height) - .map(|t| t.unwrap_or_else(CommitmentTree::empty))?; - - // Get most recent incremental witnesses for the notes we are tracking - let mut witnesses = data_guard.get_witnesses(last_height)?; - - // Get the nullifiers for the notes we are tracking - let mut nullifiers = data_guard.get_nullifiers()?; - - cache - .with_blocks( - last_height, - limit, - Box::new(|block: CompactBlock| { - let current_height = block.height(); - - // Scanned blocks MUST be height-sequential. - if current_height != (last_height + 1) { - return Err(ChainInvalid::block_height_discontinuity( - last_height + 1, - current_height, - ) - .into()); - } - - let block_hash = BlockHash::from_slice(&block.hash); - let block_time = block.time; - - let txs: Vec> = { - let mut witness_refs: Vec<_> = witnesses.iter_mut().map(|w| &mut w.1).collect(); - - scan_block( - params, - block, - &extfvks, - &nullifiers, - &mut tree, - &mut witness_refs[..], - ) - }; - - // Enforce that all roots match. This is slow, so only include in debug builds. - #[cfg(debug_assertions)] - { - let cur_root = tree.root(); - for row in &witnesses { - if row.1.root() != cur_root { - return Err(Error::InvalidWitnessAnchor(row.0, current_height).into()); - } - } - for tx in &txs { - for output in tx.shielded_outputs.iter() { - if output.witness.root() != cur_root { - return Err(Error::InvalidNewWitnessAnchor( - output.index, - tx.txid, - current_height, - output.witness.root(), - ) - .into()); - } - } - } - } - - let new_witnesses = data_guard.advance_by_block( - &(PrunedBlock { - block_height: current_height, - block_hash, - block_time, - commitment_tree: &tree, - transactions: &txs, - }), - &witnesses, - )?; - - let spent_nf: Vec = txs - .iter() - .flat_map(|tx| tx.shielded_spends.iter().map(|spend| spend.nf)) - .collect(); - nullifiers.retain(|(_, nf)| !spent_nf.contains(nf)); - nullifiers - .extend(txs.iter().flat_map(|tx| { - tx.shielded_outputs.iter().map(|out| (out.account, out.nf)) - })); - - witnesses.extend(new_witnesses); - - last_height = current_height; - - Ok(()) - }), - ) - .await -} diff --git a/zcash_client_backend/src/with_async/mod.rs b/zcash_client_backend/src/with_async/mod.rs deleted file mode 100644 index c86bf429e0..0000000000 --- a/zcash_client_backend/src/with_async/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod data_api; diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index c6b1957d06..6ff2a443db 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -25,6 +25,7 @@ rand_core = "0.5.1" rusqlite = { version = "0.28", features = ["bundled", "time"] } libsqlite3-sys= { version = "0.25.2", features = ["bundled"] } time = "0.3" +tokio = { version = "1.20", features = ["full"] } zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } zcash_primitives = { version = "0.5", path = "../zcash_primitives" } diff --git a/zcash_client_sqlite/src/with_async/mod.rs b/zcash_client_sqlite/src/with_async/mod.rs index 0c25989024..7015ee5954 100644 --- a/zcash_client_sqlite/src/with_async/mod.rs +++ b/zcash_client_sqlite/src/with_async/mod.rs @@ -16,6 +16,16 @@ use zcash_primitives::transaction::components::Amount; use zcash_primitives::transaction::TxId; use zcash_primitives::zip32::ExtendedFullViewingKey; +pub async fn async_blocking(blocking_fn: F) -> R +where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, +{ + tokio::task::spawn_blocking(blocking_fn) + .await + .expect("spawn_blocking to succeed") +} + #[async_trait::async_trait] pub trait WalletRead: Send + Sync + 'static { type Error; @@ -66,7 +76,9 @@ pub trait WalletRead: Send + Sync + 'static { /// /// This will return `Ok(None)` if no block data is present in the database. async fn get_max_height_hash(&self) -> Result, Self::Error> { + println!("before extrema"); let extrema = self.block_height_extrema().await?; + println!("extrema {extrema:?}"); let res = if let Some((_, max_height)) = extrema { self.get_block_hash(max_height) .await @@ -237,33 +249,53 @@ impl WalletRead for WalletDbAs async fn block_height_extrema( &self, ) -> Result, Self::Error> { - let db = self.inner.lock().unwrap(); - wallet::block_height_extrema(&db).map_err(SqliteClientError::from) + let db = self.clone(); + async_blocking(move || { + let db = db.inner.lock().unwrap(); + wallet::block_height_extrema(&db).map_err(SqliteClientError::from) + }) + .await } async fn get_block_hash( &self, block_height: BlockHeight, ) -> Result, Self::Error> { - let db = self.inner.lock().unwrap(); - wallet::get_block_hash(&db, block_height).map_err(SqliteClientError::from) + let db = self.clone(); + async_blocking(move || { + let db = db.inner.lock().unwrap(); + wallet::get_block_hash(&db, block_height).map_err(SqliteClientError::from) + }) + .await } async fn get_tx_height(&self, txid: TxId) -> Result, Self::Error> { - let db = self.inner.lock().unwrap(); - wallet::get_tx_height(&db, txid).map_err(SqliteClientError::from) + let db = self.clone(); + async_blocking(move || { + let db = db.inner.lock().unwrap(); + wallet::get_tx_height(&db, txid).map_err(SqliteClientError::from) + }) + .await } async fn get_address(&self, account: AccountId) -> Result, Self::Error> { - let db = self.inner.lock().unwrap(); - wallet::get_address(&db, account).map_err(SqliteClientError::from) + let db = self.clone(); + async_blocking(move || { + let db = db.inner.lock().unwrap(); + wallet::get_address(&db, account).map_err(SqliteClientError::from) + }) + .await } async fn get_extended_full_viewing_keys( &self, ) -> Result, Self::Error> { - let db = self.inner.lock().unwrap(); - wallet::get_extended_full_viewing_keys(&db) + let db = self.clone(); + async_blocking(move || { + let db = db.inner.lock().unwrap(); + wallet::get_extended_full_viewing_keys(&db).map_err(SqliteClientError::from) + }) + .await } async fn is_valid_account_extfvk( @@ -271,8 +303,13 @@ impl WalletRead for WalletDbAs account: AccountId, extfvk: &ExtendedFullViewingKey, ) -> Result { - let db = self.inner.lock().unwrap(); - wallet::is_valid_account_extfvk(&db, account, extfvk) + let db = self.clone(); + let extfvk = extfvk.clone(); + async_blocking(move || { + let db = db.inner.lock().unwrap(); + wallet::is_valid_account_extfvk(&db, account, &extfvk) + }) + .await } async fn get_balance_at( @@ -280,24 +317,36 @@ impl WalletRead for WalletDbAs account: AccountId, anchor_height: BlockHeight, ) -> Result { - let db = self.inner.lock().unwrap(); - wallet::get_balance_at(&db, account, anchor_height) + let db = self.clone(); + async_blocking(move || { + let db = db.inner.lock().unwrap(); + wallet::get_balance_at(&db, account, anchor_height) + }) + .await } async fn get_memo(&self, id_note: Self::NoteRef) -> Result { - let db = self.inner.lock().unwrap(); - match id_note { - NoteId::SentNoteId(id_note) => wallet::get_sent_memo(&db, id_note), - NoteId::ReceivedNoteId(id_note) => wallet::get_received_memo(&db, id_note), - } + let db = self.clone(); + async_blocking(move || { + let db = db.inner.lock().unwrap(); + match id_note { + NoteId::SentNoteId(id_note) => wallet::get_sent_memo(&db, id_note), + NoteId::ReceivedNoteId(id_note) => wallet::get_received_memo(&db, id_note), + } + }) + .await } async fn get_commitment_tree( &self, block_height: BlockHeight, ) -> Result>, Self::Error> { - let db = self.inner.lock().unwrap(); - wallet::get_commitment_tree(&db, block_height) + let db = self.clone(); + async_blocking(move || { + let db = db.inner.lock().unwrap(); + wallet::get_commitment_tree(&db, block_height) + }) + .await } #[allow(clippy::type_complexity)] @@ -305,13 +354,21 @@ impl WalletRead for WalletDbAs &self, block_height: BlockHeight, ) -> Result)>, Self::Error> { - let db = self.inner.lock().unwrap(); - wallet::get_witnesses(&db, block_height) + let db = self.clone(); + async_blocking(move || { + let db = db.inner.lock().unwrap(); + wallet::get_witnesses(&db, block_height) + }) + .await } async fn get_nullifiers(&self) -> Result, Self::Error> { - let db = self.inner.lock().unwrap(); - wallet::get_nullifiers(&db) + let db = self.clone(); + async_blocking(move || { + let db = db.inner.lock().unwrap(); + wallet::get_nullifiers(&db) + }) + .await } async fn get_spendable_notes( @@ -319,8 +376,12 @@ impl WalletRead for WalletDbAs account: AccountId, anchor_height: BlockHeight, ) -> Result, Self::Error> { - let db = self.inner.lock().unwrap(); - wallet::transact::get_spendable_notes(&db, account, anchor_height) + let db = self.clone(); + async_blocking(move || { + let db = db.inner.lock().unwrap(); + wallet::transact::get_spendable_notes(&db, account, anchor_height) + }) + .await } async fn select_spendable_notes( @@ -329,11 +390,16 @@ impl WalletRead for WalletDbAs target_value: Amount, anchor_height: BlockHeight, ) -> Result, Self::Error> { - let db = self.inner.lock().unwrap(); - wallet::transact::select_spendable_notes(&db, account, target_value, anchor_height) + let db = self.clone(); + async_blocking(move || { + let db = db.inner.lock().unwrap(); + wallet::transact::select_spendable_notes(&db, account, target_value, anchor_height) + }) + .await } } +#[derive(Clone)] pub struct DataConnStmtCacheAsync

{ wallet_db: WalletDbAsync

, } @@ -361,16 +427,16 @@ impl DataConnStmtCacheAsync

{ } Err(error) => { match self.wallet_db.inner.lock().unwrap().conn.execute("ROLLBACK", []) { - Ok(_) => Err(error), - Err(e) => - // Panicking here is probably the right thing to do, because it - // means the database is corrupt. - panic!( - "Rollback failed with error {} while attempting to recover from error {}; database is likely corrupt.", - e, - error - ) - } + Ok(_) => Err(error), + Err(e) => + // Panicking here is probably the right thing to do, because it + // means the database is corrupt. + panic!( + "Rollback failed with error {} while attempting to recover from error {}; database is likely corrupt.", + e, + error + ) + } } } } @@ -385,28 +451,28 @@ impl WalletRead for DataConnSt async fn block_height_extrema( &self, ) -> Result, Self::Error> { - self.block_height_extrema().await + self.wallet_db.block_height_extrema().await } async fn get_block_hash( &self, block_height: BlockHeight, ) -> Result, Self::Error> { - self.get_block_hash(block_height).await + self.wallet_db.get_block_hash(block_height).await } async fn get_tx_height(&self, txid: TxId) -> Result, Self::Error> { - self.get_tx_height(txid).await + self.wallet_db.get_tx_height(txid).await } async fn get_address(&self, account: AccountId) -> Result, Self::Error> { - self.get_address(account).await + self.wallet_db.get_address(account).await } async fn get_extended_full_viewing_keys( &self, ) -> Result, Self::Error> { - self.get_extended_full_viewing_keys().await + self.wallet_db.get_extended_full_viewing_keys().await } async fn is_valid_account_extfvk( @@ -414,7 +480,9 @@ impl WalletRead for DataConnSt account: AccountId, extfvk: &ExtendedFullViewingKey, ) -> Result { - self.is_valid_account_extfvk(account, extfvk).await + self.wallet_db + .is_valid_account_extfvk(account, extfvk) + .await } async fn get_balance_at( @@ -422,18 +490,18 @@ impl WalletRead for DataConnSt account: AccountId, anchor_height: BlockHeight, ) -> Result { - self.get_balance_at(account, anchor_height).await + self.wallet_db.get_balance_at(account, anchor_height).await } async fn get_memo(&self, id_note: Self::NoteRef) -> Result { - self.get_memo(id_note).await + self.wallet_db.get_memo(id_note).await } async fn get_commitment_tree( &self, block_height: BlockHeight, ) -> Result>, Self::Error> { - self.get_commitment_tree(block_height).await + self.wallet_db.get_commitment_tree(block_height).await } #[allow(clippy::type_complexity)] @@ -441,11 +509,11 @@ impl WalletRead for DataConnSt &self, block_height: BlockHeight, ) -> Result)>, Self::Error> { - self.get_witnesses(block_height).await + self.wallet_db.get_witnesses(block_height).await } async fn get_nullifiers(&self) -> Result, Self::Error> { - self.get_nullifiers().await + self.wallet_db.get_nullifiers().await } async fn get_spendable_notes( @@ -453,7 +521,9 @@ impl WalletRead for DataConnSt account: AccountId, anchor_height: BlockHeight, ) -> Result, Self::Error> { - self.get_spendable_notes(account, anchor_height).await + self.wallet_db + .get_spendable_notes(account, anchor_height) + .await } async fn select_spendable_notes( @@ -462,7 +532,8 @@ impl WalletRead for DataConnSt target_value: Amount, anchor_height: BlockHeight, ) -> Result, Self::Error> { - self.select_spendable_notes(account, target_value, anchor_height) + self.wallet_db + .select_spendable_notes(account, target_value, anchor_height) .await } } @@ -586,7 +657,11 @@ impl WalletWrite for DataConnS } async fn rewind_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error> { - let db = self.wallet_db.inner.lock().unwrap(); - wallet::rewind_to_height(&db, block_height) + let db = self.clone(); + async_blocking(move || { + let db = db.wallet_db.inner.lock().unwrap(); + wallet::rewind_to_height(&db, block_height) + }) + .await } } diff --git a/zcash_client_sqlite/src/with_async/wallet_actions.rs b/zcash_client_sqlite/src/with_async/wallet_actions.rs index 6465702319..924ca8d770 100644 --- a/zcash_client_sqlite/src/with_async/wallet_actions.rs +++ b/zcash_client_sqlite/src/with_async/wallet_actions.rs @@ -1,6 +1,6 @@ use crate::error::SqliteClientError; use crate::wallet::ShieldedOutput; -use crate::with_async::WalletDbAsync; +use crate::with_async::{async_blocking, WalletDbAsync}; use crate::{NoteId, WalletDb}; use ff::PrimeField; use rusqlite::{params, ToSql}; @@ -27,8 +27,8 @@ pub fn insert_block

( commitment_tree: &CommitmentTree, ) -> Result<(), SqliteClientError> { let mut encoded_tree = Vec::new(); - commitment_tree.write(&mut encoded_tree).unwrap(); + commitment_tree.write(&mut encoded_tree).unwrap(); db.conn .prepare( "INSERT INTO blocks (height, hash, time, sapling_tree) From ecb0295a903dc90d07fb7f13d327a4673f97a37b Mon Sep 17 00:00:00 2001 From: borngraced Date: Sun, 27 Aug 2023 13:13:47 +0100 Subject: [PATCH 10/42] =?UTF-8?q?save=20dev=20state=20=E2=80=94=20finish?= =?UTF-8?q?=20modification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- zcash_client_sqlite/src/with_async/mod.rs | 12 ++++----- .../src/with_async/wallet_actions.rs | 25 +++++++++---------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/zcash_client_sqlite/src/with_async/mod.rs b/zcash_client_sqlite/src/with_async/mod.rs index 7015ee5954..aaf31315b1 100644 --- a/zcash_client_sqlite/src/with_async/mod.rs +++ b/zcash_client_sqlite/src/with_async/mod.rs @@ -204,11 +204,9 @@ pub trait WalletWrite: WalletRead { use crate::error::SqliteClientError; use crate::{wallet, NoteId, WalletDb}; -use rusqlite::{Connection, OptionalExtension, Statement, ToSql}; +use rusqlite::Connection; use std::sync::{Arc, Mutex}; -use zcash_client_backend::encoding::{ - decode_extended_full_viewing_key, decode_payment_address, encode_extended_full_viewing_key, -}; + use zcash_primitives::consensus; /// A wrapper for the SQLite connection to the wallet database. @@ -555,12 +553,12 @@ impl WalletWrite for DataConnS block.block_height, block.block_hash, block.block_time, - &block.commitment_tree, + block.commitment_tree, )?; let mut new_witnesses = vec![]; for tx in block.transactions { - let tx_row = wallet_actions::put_tx_meta(&db, &tx, block.block_height)?; + let tx_row = wallet_actions::put_tx_meta(&db, tx, block.block_height)?; // Mark notes as spent and remove them from the scanning cache for spend in &tx.shielded_spends { @@ -627,7 +625,7 @@ impl WalletWrite for DataConnS // Update the database atomically, to ensure the result is internally consistent. self.transactionally(|up| { let db = up.wallet_db.inner.lock().unwrap(); - let tx_ref = wallet_actions::put_tx_data(&db, &sent_tx.tx, Some(sent_tx.created))?; + let tx_ref = wallet_actions::put_tx_data(&db, sent_tx.tx, Some(sent_tx.created))?; // Mark notes as spent. // diff --git a/zcash_client_sqlite/src/with_async/wallet_actions.rs b/zcash_client_sqlite/src/with_async/wallet_actions.rs index 924ca8d770..babb1450a9 100644 --- a/zcash_client_sqlite/src/with_async/wallet_actions.rs +++ b/zcash_client_sqlite/src/with_async/wallet_actions.rs @@ -1,6 +1,5 @@ use crate::error::SqliteClientError; use crate::wallet::ShieldedOutput; -use crate::with_async::{async_blocking, WalletDbAsync}; use crate::{NoteId, WalletDb}; use ff::PrimeField; use rusqlite::{params, ToSql}; @@ -84,7 +83,7 @@ pub fn put_tx_meta( /// /// Marking a note spent in this fashion does NOT imply that the /// spending transaction has been mined. -pub fn mark_spent<'a, P>( +pub fn mark_spent

( db: &MutexGuard>, tx_ref: i64, nf: &Nullifier, @@ -116,15 +115,15 @@ pub fn put_received_note( let nf_bytes = output.nullifier().map(|nf| nf.0.to_vec()); let sql_args: &[(&str, &dyn ToSql)] = &[ - (&":account", &account), - (&":diversifier", &diversifier), - (&":value", &value), - (&":rcm", &rcm), - (&":nf", &nf_bytes), - (&":memo", &memo), - (&":is_change", &is_change), - (&":tx", &tx), - (&":output_index", &output_index), + (":account", &account), + (":diversifier", &diversifier), + (":value", &value), + (":rcm", &rcm), + (":nf", &nf_bytes), + (":memo", &memo), + (":is_change", &is_change), + (":tx", &tx), + (":output_index", &output_index), ]; // First try updating an existing received note into the database. @@ -141,7 +140,7 @@ pub fn put_received_note( is_change = IFNULL(:is_change, is_change) WHERE tx = :tx AND output_index = :output_index", )? - .execute_named(&sql_args)? + .execute(sql_args)? == 0 { // It isn't there, so insert our note into the database. @@ -157,7 +156,7 @@ pub fn put_received_note( is_change = IFNULL(:is_change, is_change) WHERE tx = :tx AND output_index = :output_index", )? - .execute_named(&sql_args)?; + .execute(sql_args)?; Ok(NoteId::ReceivedNoteId(db.conn.last_insert_rowid())) } else { From 624ed96642514ade93b57dc86c3d47f0d3fd188d Mon Sep 17 00:00:00 2001 From: borngraced Date: Sun, 27 Aug 2023 14:05:50 +0100 Subject: [PATCH 11/42] =?UTF-8?q?save=20dev=20state=20=E2=80=94=20minor=20?= =?UTF-8?q?changes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.toml | 1 + zcash_client_sqlite/Cargo.toml | 2 + zcash_client_sqlite/src/with_async/mod.rs | 180 +--------------------- 3 files changed, 4 insertions(+), 179 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 846f8c7a7b..78e6fdb3b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "zcash_client_backend", "zcash_client_sqlite", "zcash_extensions", + "zcash_extras", "zcash_history", "zcash_primitives", "zcash_proofs", diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 6ff2a443db..7779855212 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -27,8 +27,10 @@ libsqlite3-sys= { version = "0.25.2", features = ["bundled"] } time = "0.3" tokio = { version = "1.20", features = ["full"] } zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } +zcash_extras = { version = "0.1", path = "../zcash_extras" } zcash_primitives = { version = "0.5", path = "../zcash_primitives" } + [dev-dependencies] rand_core = "0.5.1" tempfile = "3" diff --git a/zcash_client_sqlite/src/with_async/mod.rs b/zcash_client_sqlite/src/with_async/mod.rs index aaf31315b1..3fa590dbb9 100644 --- a/zcash_client_sqlite/src/with_async/mod.rs +++ b/zcash_client_sqlite/src/with_async/mod.rs @@ -1,12 +1,10 @@ pub mod wallet_actions; -use std::cmp; use std::collections::HashMap; -use std::fmt::Debug; use std::path::Path; -use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET; use zcash_client_backend::data_api::{PrunedBlock, ReceivedTransaction, SentTransaction}; use zcash_client_backend::wallet::{AccountId, SpendableNote}; +use zcash_extras::{WalletRead, WalletWrite}; use zcash_primitives::block::BlockHash; use zcash_primitives::consensus::BlockHeight; use zcash_primitives::memo::Memo; @@ -26,182 +24,6 @@ where .expect("spawn_blocking to succeed") } -#[async_trait::async_trait] -pub trait WalletRead: Send + Sync + 'static { - type Error; - type NoteRef: Copy + Debug; - type TxRef: Copy + Debug; - - /// Returns the minimum and maximum block heights for stored blocks. - /// - /// This will return `Ok(None)` if no block data is present in the database. - async fn block_height_extrema(&self) - -> Result, Self::Error>; - - /// Returns the default target height (for the block in which a new - /// transaction would be mined) and anchor height (to use for a new - /// transaction), given the range of block heights that the backend - /// knows about. - /// - /// This will return `Ok(None)` if no block data is present in the database. - async fn get_target_and_anchor_heights( - &self, - ) -> Result, Self::Error> { - self.block_height_extrema().await.map(|heights| { - heights.map(|(min_height, max_height)| { - let target_height = max_height + 1; - - // Select an anchor ANCHOR_OFFSET back from the target block, - // unless that would be before the earliest block we have. - let anchor_height = BlockHeight::from(cmp::max( - u32::from(target_height).saturating_sub(ANCHOR_OFFSET), - u32::from(min_height), - )); - - (target_height, anchor_height) - }) - }) - } - - /// Returns the block hash for the block at the given height, if the - /// associated block data is available. Returns `Ok(None)` if the hash - /// is not found in the database. - async fn get_block_hash( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error>; - - /// Returns the block hash for the block at the maximum height known - /// in stored data. - /// - /// This will return `Ok(None)` if no block data is present in the database. - async fn get_max_height_hash(&self) -> Result, Self::Error> { - println!("before extrema"); - let extrema = self.block_height_extrema().await?; - println!("extrema {extrema:?}"); - let res = if let Some((_, max_height)) = extrema { - self.get_block_hash(max_height) - .await - .map(|hash_opt| hash_opt.map(move |hash| (max_height, hash)))? - } else { - None - }; - - Ok(res) - } - - /// Returns the block height in which the specified transaction was mined, - /// or `Ok(None)` if the transaction is not mined in the main chain. - async fn get_tx_height(&self, txid: TxId) -> Result, Self::Error>; - - /// Returns the payment address for the specified account, if the account - /// identifier specified refers to a valid account for this wallet. - /// - /// This will return `Ok(None)` if the account identifier does not correspond - /// to a known account. - async fn get_address(&self, account: AccountId) -> Result, Self::Error>; - - /// Returns all extended full viewing keys known about by this wallet. - async fn get_extended_full_viewing_keys( - &self, - ) -> Result, Self::Error>; - - /// Checks whether the specified extended full viewing key is - /// associated with the account. - async fn is_valid_account_extfvk( - &self, - account: AccountId, - extfvk: &ExtendedFullViewingKey, - ) -> Result; - - /// Returns the wallet balance for an account as of the specified block - /// height. - /// - /// This may be used to obtain a balance that ignores notes that have been - /// received so recently that they are not yet deemed spendable. - async fn get_balance_at( - &self, - account: AccountId, - anchor_height: BlockHeight, - ) -> Result; - - /// Returns the memo for a note. - /// - /// Implementations of this method must return an error if the note identifier - /// does not appear in the backing data store. - async fn get_memo(&self, id_note: Self::NoteRef) -> Result; - - /// Returns the note commitment tree at the specified block height. - async fn get_commitment_tree( - &self, - block_height: BlockHeight, - ) -> Result>, Self::Error>; - - /// Returns the incremental witnesses as of the specified block height. - #[allow(clippy::type_complexity)] - async fn get_witnesses( - &self, - block_height: BlockHeight, - ) -> Result)>, Self::Error>; - - /// Returns the unspent nullifiers, along with the account identifiers - /// with which they are associated. - async fn get_nullifiers(&self) -> Result, Self::Error>; - - /// Return all spendable notes. - async fn get_spendable_notes( - &self, - account: AccountId, - anchor_height: BlockHeight, - ) -> Result, Self::Error>; - - /// Returns a list of spendable notes sufficient to cover the specified - /// target value, if possible. - async fn select_spendable_notes( - &self, - account: AccountId, - target_value: Amount, - anchor_height: BlockHeight, - ) -> Result, Self::Error>; -} - -/// This trait encapsulates the write capabilities required to update stored -/// wallet data. -#[async_trait::async_trait] -pub trait WalletWrite: WalletRead { - #[allow(clippy::type_complexity)] - async fn advance_by_block( - &mut self, - block: &PrunedBlock, - updated_witnesses: &[(Self::NoteRef, IncrementalWitness)], - ) -> Result)>, Self::Error>; - - async fn store_received_tx( - &mut self, - received_tx: &ReceivedTransaction, - ) -> Result; - - async fn store_sent_tx( - &mut self, - sent_tx: &SentTransaction, - ) -> Result; - - /// Rewinds the wallet database to the specified height. - /// - /// This method assumes that the state of the underlying data store is - /// consistent up to a particular block height. Since it is possible that - /// a chain reorg might invalidate some stored state, this method must be - /// implemented in order to allow users of this API to "reset" the data store - /// to correctly represent chainstate as of a specified block height. - /// - /// After calling this method, the block at the given height will be the - /// most recent block and all other operations will treat this block - /// as the chain tip for balance determination purposes. - /// - /// There may be restrictions on how far it is possible to rewind. - async fn rewind_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error>; -} - use crate::error::SqliteClientError; use crate::{wallet, NoteId, WalletDb}; use rusqlite::Connection; From e473d796598b04344e5875bae3b78b1c90a79920 Mon Sep 17 00:00:00 2001 From: borngraced Date: Sun, 27 Aug 2023 14:05:50 +0100 Subject: [PATCH 12/42] =?UTF-8?q?save=20dev=20state=20=E2=80=94=20zcash=20?= =?UTF-8?q?extras?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.toml | 1 + zcash_client_sqlite/Cargo.toml | 2 + zcash_client_sqlite/src/with_async/mod.rs | 180 +------------------- zcash_extras/Cargo.toml | 11 ++ zcash_extras/src/lib.rs | 196 ++++++++++++++++++++++ 5 files changed, 211 insertions(+), 179 deletions(-) create mode 100644 zcash_extras/Cargo.toml create mode 100644 zcash_extras/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 846f8c7a7b..78e6fdb3b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "zcash_client_backend", "zcash_client_sqlite", "zcash_extensions", + "zcash_extras", "zcash_history", "zcash_primitives", "zcash_proofs", diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 6ff2a443db..7779855212 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -27,8 +27,10 @@ libsqlite3-sys= { version = "0.25.2", features = ["bundled"] } time = "0.3" tokio = { version = "1.20", features = ["full"] } zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } +zcash_extras = { version = "0.1", path = "../zcash_extras" } zcash_primitives = { version = "0.5", path = "../zcash_primitives" } + [dev-dependencies] rand_core = "0.5.1" tempfile = "3" diff --git a/zcash_client_sqlite/src/with_async/mod.rs b/zcash_client_sqlite/src/with_async/mod.rs index aaf31315b1..3fa590dbb9 100644 --- a/zcash_client_sqlite/src/with_async/mod.rs +++ b/zcash_client_sqlite/src/with_async/mod.rs @@ -1,12 +1,10 @@ pub mod wallet_actions; -use std::cmp; use std::collections::HashMap; -use std::fmt::Debug; use std::path::Path; -use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET; use zcash_client_backend::data_api::{PrunedBlock, ReceivedTransaction, SentTransaction}; use zcash_client_backend::wallet::{AccountId, SpendableNote}; +use zcash_extras::{WalletRead, WalletWrite}; use zcash_primitives::block::BlockHash; use zcash_primitives::consensus::BlockHeight; use zcash_primitives::memo::Memo; @@ -26,182 +24,6 @@ where .expect("spawn_blocking to succeed") } -#[async_trait::async_trait] -pub trait WalletRead: Send + Sync + 'static { - type Error; - type NoteRef: Copy + Debug; - type TxRef: Copy + Debug; - - /// Returns the minimum and maximum block heights for stored blocks. - /// - /// This will return `Ok(None)` if no block data is present in the database. - async fn block_height_extrema(&self) - -> Result, Self::Error>; - - /// Returns the default target height (for the block in which a new - /// transaction would be mined) and anchor height (to use for a new - /// transaction), given the range of block heights that the backend - /// knows about. - /// - /// This will return `Ok(None)` if no block data is present in the database. - async fn get_target_and_anchor_heights( - &self, - ) -> Result, Self::Error> { - self.block_height_extrema().await.map(|heights| { - heights.map(|(min_height, max_height)| { - let target_height = max_height + 1; - - // Select an anchor ANCHOR_OFFSET back from the target block, - // unless that would be before the earliest block we have. - let anchor_height = BlockHeight::from(cmp::max( - u32::from(target_height).saturating_sub(ANCHOR_OFFSET), - u32::from(min_height), - )); - - (target_height, anchor_height) - }) - }) - } - - /// Returns the block hash for the block at the given height, if the - /// associated block data is available. Returns `Ok(None)` if the hash - /// is not found in the database. - async fn get_block_hash( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error>; - - /// Returns the block hash for the block at the maximum height known - /// in stored data. - /// - /// This will return `Ok(None)` if no block data is present in the database. - async fn get_max_height_hash(&self) -> Result, Self::Error> { - println!("before extrema"); - let extrema = self.block_height_extrema().await?; - println!("extrema {extrema:?}"); - let res = if let Some((_, max_height)) = extrema { - self.get_block_hash(max_height) - .await - .map(|hash_opt| hash_opt.map(move |hash| (max_height, hash)))? - } else { - None - }; - - Ok(res) - } - - /// Returns the block height in which the specified transaction was mined, - /// or `Ok(None)` if the transaction is not mined in the main chain. - async fn get_tx_height(&self, txid: TxId) -> Result, Self::Error>; - - /// Returns the payment address for the specified account, if the account - /// identifier specified refers to a valid account for this wallet. - /// - /// This will return `Ok(None)` if the account identifier does not correspond - /// to a known account. - async fn get_address(&self, account: AccountId) -> Result, Self::Error>; - - /// Returns all extended full viewing keys known about by this wallet. - async fn get_extended_full_viewing_keys( - &self, - ) -> Result, Self::Error>; - - /// Checks whether the specified extended full viewing key is - /// associated with the account. - async fn is_valid_account_extfvk( - &self, - account: AccountId, - extfvk: &ExtendedFullViewingKey, - ) -> Result; - - /// Returns the wallet balance for an account as of the specified block - /// height. - /// - /// This may be used to obtain a balance that ignores notes that have been - /// received so recently that they are not yet deemed spendable. - async fn get_balance_at( - &self, - account: AccountId, - anchor_height: BlockHeight, - ) -> Result; - - /// Returns the memo for a note. - /// - /// Implementations of this method must return an error if the note identifier - /// does not appear in the backing data store. - async fn get_memo(&self, id_note: Self::NoteRef) -> Result; - - /// Returns the note commitment tree at the specified block height. - async fn get_commitment_tree( - &self, - block_height: BlockHeight, - ) -> Result>, Self::Error>; - - /// Returns the incremental witnesses as of the specified block height. - #[allow(clippy::type_complexity)] - async fn get_witnesses( - &self, - block_height: BlockHeight, - ) -> Result)>, Self::Error>; - - /// Returns the unspent nullifiers, along with the account identifiers - /// with which they are associated. - async fn get_nullifiers(&self) -> Result, Self::Error>; - - /// Return all spendable notes. - async fn get_spendable_notes( - &self, - account: AccountId, - anchor_height: BlockHeight, - ) -> Result, Self::Error>; - - /// Returns a list of spendable notes sufficient to cover the specified - /// target value, if possible. - async fn select_spendable_notes( - &self, - account: AccountId, - target_value: Amount, - anchor_height: BlockHeight, - ) -> Result, Self::Error>; -} - -/// This trait encapsulates the write capabilities required to update stored -/// wallet data. -#[async_trait::async_trait] -pub trait WalletWrite: WalletRead { - #[allow(clippy::type_complexity)] - async fn advance_by_block( - &mut self, - block: &PrunedBlock, - updated_witnesses: &[(Self::NoteRef, IncrementalWitness)], - ) -> Result)>, Self::Error>; - - async fn store_received_tx( - &mut self, - received_tx: &ReceivedTransaction, - ) -> Result; - - async fn store_sent_tx( - &mut self, - sent_tx: &SentTransaction, - ) -> Result; - - /// Rewinds the wallet database to the specified height. - /// - /// This method assumes that the state of the underlying data store is - /// consistent up to a particular block height. Since it is possible that - /// a chain reorg might invalidate some stored state, this method must be - /// implemented in order to allow users of this API to "reset" the data store - /// to correctly represent chainstate as of a specified block height. - /// - /// After calling this method, the block at the given height will be the - /// most recent block and all other operations will treat this block - /// as the chain tip for balance determination purposes. - /// - /// There may be restrictions on how far it is possible to rewind. - async fn rewind_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error>; -} - use crate::error::SqliteClientError; use crate::{wallet, NoteId, WalletDb}; use rusqlite::Connection; diff --git a/zcash_extras/Cargo.toml b/zcash_extras/Cargo.toml new file mode 100644 index 0000000000..af335c77e2 --- /dev/null +++ b/zcash_extras/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "zcash_extras" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +async-trait = "0.1.52" +zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } +zcash_primitives = { version = "0.5", path = "../zcash_primitives" } diff --git a/zcash_extras/src/lib.rs b/zcash_extras/src/lib.rs new file mode 100644 index 0000000000..471687d123 --- /dev/null +++ b/zcash_extras/src/lib.rs @@ -0,0 +1,196 @@ +use std::cmp; +use std::collections::HashMap; +use std::fmt::Debug; +use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET; +use zcash_client_backend::data_api::PrunedBlock; +use zcash_client_backend::data_api::ReceivedTransaction; +use zcash_client_backend::data_api::SentTransaction; +use zcash_client_backend::wallet::AccountId; +use zcash_client_backend::wallet::SpendableNote; +use zcash_primitives::block::BlockHash; +use zcash_primitives::consensus::BlockHeight; +use zcash_primitives::memo::Memo; +use zcash_primitives::merkle_tree::CommitmentTree; +use zcash_primitives::merkle_tree::IncrementalWitness; +use zcash_primitives::sapling::Node; +use zcash_primitives::sapling::Nullifier; +use zcash_primitives::sapling::PaymentAddress; +use zcash_primitives::transaction::components::Amount; +use zcash_primitives::transaction::TxId; +use zcash_primitives::zip32::ExtendedFullViewingKey; + +#[async_trait::async_trait] +pub trait WalletRead: Send + Sync + 'static { + type Error; + type NoteRef: Copy + Debug; + type TxRef: Copy + Debug; + + /// Returns the minimum and maximum block heights for stored blocks. + /// + /// This will return `Ok(None)` if no block data is present in the database. + async fn block_height_extrema(&self) + -> Result, Self::Error>; + + /// Returns the default target height (for the block in which a new + /// transaction would be mined) and anchor height (to use for a new + /// transaction), given the range of block heights that the backend + /// knows about. + /// + /// This will return `Ok(None)` if no block data is present in the database. + async fn get_target_and_anchor_heights( + &self, + ) -> Result, Self::Error> { + self.block_height_extrema().await.map(|heights| { + heights.map(|(min_height, max_height)| { + let target_height = max_height + 1; + + // Select an anchor ANCHOR_OFFSET back from the target block, + // unless that would be before the earliest block we have. + let anchor_height = BlockHeight::from(cmp::max( + u32::from(target_height).saturating_sub(ANCHOR_OFFSET), + u32::from(min_height), + )); + + (target_height, anchor_height) + }) + }) + } + + /// Returns the block hash for the block at the given height, if the + /// associated block data is available. Returns `Ok(None)` if the hash + /// is not found in the database. + async fn get_block_hash( + &self, + block_height: BlockHeight, + ) -> Result, Self::Error>; + + /// Returns the block hash for the block at the maximum height known + /// in stored data. + /// + /// This will return `Ok(None)` if no block data is present in the database. + async fn get_max_height_hash(&self) -> Result, Self::Error> { + println!("before extrema"); + let extrema = self.block_height_extrema().await?; + println!("extrema {extrema:?}"); + let res = if let Some((_, max_height)) = extrema { + self.get_block_hash(max_height) + .await + .map(|hash_opt| hash_opt.map(move |hash| (max_height, hash)))? + } else { + None + }; + + Ok(res) + } + + /// Returns the block height in which the specified transaction was mined, + /// or `Ok(None)` if the transaction is not mined in the main chain. + async fn get_tx_height(&self, txid: TxId) -> Result, Self::Error>; + + /// Returns the payment address for the specified account, if the account + /// identifier specified refers to a valid account for this wallet. + /// + /// This will return `Ok(None)` if the account identifier does not correspond + /// to a known account. + async fn get_address(&self, account: AccountId) -> Result, Self::Error>; + + /// Returns all extended full viewing keys known about by this wallet. + async fn get_extended_full_viewing_keys( + &self, + ) -> Result, Self::Error>; + + /// Checks whether the specified extended full viewing key is + /// associated with the account. + async fn is_valid_account_extfvk( + &self, + account: AccountId, + extfvk: &ExtendedFullViewingKey, + ) -> Result; + + /// Returns the wallet balance for an account as of the specified block + /// height. + /// + /// This may be used to obtain a balance that ignores notes that have been + /// received so recently that they are not yet deemed spendable. + async fn get_balance_at( + &self, + account: AccountId, + anchor_height: BlockHeight, + ) -> Result; + + /// Returns the memo for a note. + /// + /// Implementations of this method must return an error if the note identifier + /// does not appear in the backing data store. + async fn get_memo(&self, id_note: Self::NoteRef) -> Result; + + /// Returns the note commitment tree at the specified block height. + async fn get_commitment_tree( + &self, + block_height: BlockHeight, + ) -> Result>, Self::Error>; + + /// Returns the incremental witnesses as of the specified block height. + #[allow(clippy::type_complexity)] + async fn get_witnesses( + &self, + block_height: BlockHeight, + ) -> Result)>, Self::Error>; + + /// Returns the unspent nullifiers, along with the account identifiers + /// with which they are associated. + async fn get_nullifiers(&self) -> Result, Self::Error>; + + /// Return all spendable notes. + async fn get_spendable_notes( + &self, + account: AccountId, + anchor_height: BlockHeight, + ) -> Result, Self::Error>; + + /// Returns a list of spendable notes sufficient to cover the specified + /// target value, if possible. + async fn select_spendable_notes( + &self, + account: AccountId, + target_value: Amount, + anchor_height: BlockHeight, + ) -> Result, Self::Error>; +} + +/// This trait encapsulates the write capabilities required to update stored +/// wallet data. +#[async_trait::async_trait] +pub trait WalletWrite: WalletRead { + #[allow(clippy::type_complexity)] + async fn advance_by_block( + &mut self, + block: &PrunedBlock, + updated_witnesses: &[(Self::NoteRef, IncrementalWitness)], + ) -> Result)>, Self::Error>; + + async fn store_received_tx( + &mut self, + received_tx: &ReceivedTransaction, + ) -> Result; + + async fn store_sent_tx( + &mut self, + sent_tx: &SentTransaction, + ) -> Result; + + /// Rewinds the wallet database to the specified height. + /// + /// This method assumes that the state of the underlying data store is + /// consistent up to a particular block height. Since it is possible that + /// a chain reorg might invalidate some stored state, this method must be + /// implemented in order to allow users of this API to "reset" the data store + /// to correctly represent chainstate as of a specified block height. + /// + /// After calling this method, the block at the given height will be the + /// most recent block and all other operations will treat this block + /// as the chain tip for balance determination purposes. + /// + /// There may be restrictions on how far it is possible to rewind. + async fn rewind_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error>; +} From bafac3a8582346bce2c92b6854e509240db88f54 Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 31 Aug 2023 15:35:43 +0100 Subject: [PATCH 13/42] make NoteRef not Copy --- zcash_extras/src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/zcash_extras/src/lib.rs b/zcash_extras/src/lib.rs index 471687d123..3cb684e1c7 100644 --- a/zcash_extras/src/lib.rs +++ b/zcash_extras/src/lib.rs @@ -22,7 +22,7 @@ use zcash_primitives::zip32::ExtendedFullViewingKey; #[async_trait::async_trait] pub trait WalletRead: Send + Sync + 'static { type Error; - type NoteRef: Copy + Debug; + type NoteRef: Debug; type TxRef: Copy + Debug; /// Returns the minimum and maximum block heights for stored blocks. @@ -69,9 +69,7 @@ pub trait WalletRead: Send + Sync + 'static { /// /// This will return `Ok(None)` if no block data is present in the database. async fn get_max_height_hash(&self) -> Result, Self::Error> { - println!("before extrema"); let extrema = self.block_height_extrema().await?; - println!("extrema {extrema:?}"); let res = if let Some((_, max_height)) = extrema { self.get_block_hash(max_height) .await From 3c21128cd155802355b68606f00e87e09cc41e2c Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 31 Aug 2023 17:40:12 +0100 Subject: [PATCH 14/42] move ShieldedOutput --- zcash_client_sqlite/src/wallet.rs | 65 +----------------- .../src/with_async/wallet_actions.rs | 2 +- zcash_extras/src/lib.rs | 67 ++++++++++++++++++- 3 files changed, 68 insertions(+), 66 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index c134417da2..a983f72af9 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -17,7 +17,7 @@ use zcash_primitives::{ consensus::{self, BlockHeight, NetworkUpgrade}, memo::{Memo, MemoBytes}, merkle_tree::{CommitmentTree, IncrementalWitness}, - sapling::{Node, Note, Nullifier, PaymentAddress}, + sapling::{Node, Nullifier, PaymentAddress}, transaction::{components::Amount, Transaction, TxId}, zip32::ExtendedFullViewingKey, }; @@ -29,75 +29,16 @@ use zcash_client_backend::{ decode_extended_full_viewing_key, decode_payment_address, encode_extended_full_viewing_key, encode_payment_address, }, - wallet::{AccountId, WalletShieldedOutput, WalletTx}, + wallet::{AccountId, WalletTx}, DecryptedOutput, }; +use zcash_extras::ShieldedOutput; use crate::{error::SqliteClientError, DataConnStmtCache, NoteId, WalletDb}; pub mod init; pub mod transact; -/// This trait provides a generalization over shielded output representations. -pub trait ShieldedOutput { - fn index(&self) -> usize; - fn account(&self) -> AccountId; - fn to(&self) -> &PaymentAddress; - fn note(&self) -> &Note; - fn memo(&self) -> Option<&MemoBytes>; - fn is_change(&self) -> Option; - fn nullifier(&self) -> Option; -} - -impl ShieldedOutput for WalletShieldedOutput { - fn index(&self) -> usize { - self.index - } - fn account(&self) -> AccountId { - self.account - } - fn to(&self) -> &PaymentAddress { - &self.to - } - fn note(&self) -> &Note { - &self.note - } - fn memo(&self) -> Option<&MemoBytes> { - None - } - fn is_change(&self) -> Option { - Some(self.is_change) - } - - fn nullifier(&self) -> Option { - Some(self.nf) - } -} - -impl ShieldedOutput for DecryptedOutput { - fn index(&self) -> usize { - self.index - } - fn account(&self) -> AccountId { - self.account - } - fn to(&self) -> &PaymentAddress { - &self.to - } - fn note(&self) -> &Note { - &self.note - } - fn memo(&self) -> Option<&MemoBytes> { - Some(&self.memo) - } - fn is_change(&self) -> Option { - None - } - fn nullifier(&self) -> Option { - None - } -} - /// Returns the address for the account. /// /// # Examples diff --git a/zcash_client_sqlite/src/with_async/wallet_actions.rs b/zcash_client_sqlite/src/with_async/wallet_actions.rs index babb1450a9..08a6bffb23 100644 --- a/zcash_client_sqlite/src/with_async/wallet_actions.rs +++ b/zcash_client_sqlite/src/with_async/wallet_actions.rs @@ -1,5 +1,4 @@ use crate::error::SqliteClientError; -use crate::wallet::ShieldedOutput; use crate::{NoteId, WalletDb}; use ff::PrimeField; use rusqlite::{params, ToSql}; @@ -8,6 +7,7 @@ use zcash_client_backend::address::RecipientAddress; use zcash_client_backend::encoding::encode_payment_address; use zcash_client_backend::wallet::{AccountId, WalletTx}; use zcash_client_backend::DecryptedOutput; +use zcash_extras::ShieldedOutput; use zcash_primitives::block::BlockHash; use zcash_primitives::consensus; use zcash_primitives::consensus::BlockHeight; diff --git a/zcash_extras/src/lib.rs b/zcash_extras/src/lib.rs index 3cb684e1c7..d4739d4cc5 100644 --- a/zcash_extras/src/lib.rs +++ b/zcash_extras/src/lib.rs @@ -5,16 +5,17 @@ use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET; use zcash_client_backend::data_api::PrunedBlock; use zcash_client_backend::data_api::ReceivedTransaction; use zcash_client_backend::data_api::SentTransaction; -use zcash_client_backend::wallet::AccountId; use zcash_client_backend::wallet::SpendableNote; +use zcash_client_backend::wallet::{AccountId, WalletShieldedOutput}; +use zcash_client_backend::DecryptedOutput; use zcash_primitives::block::BlockHash; use zcash_primitives::consensus::BlockHeight; -use zcash_primitives::memo::Memo; +use zcash_primitives::memo::{Memo, MemoBytes}; use zcash_primitives::merkle_tree::CommitmentTree; use zcash_primitives::merkle_tree::IncrementalWitness; -use zcash_primitives::sapling::Node; use zcash_primitives::sapling::Nullifier; use zcash_primitives::sapling::PaymentAddress; +use zcash_primitives::sapling::{Node, Note}; use zcash_primitives::transaction::components::Amount; use zcash_primitives::transaction::TxId; use zcash_primitives::zip32::ExtendedFullViewingKey; @@ -192,3 +193,63 @@ pub trait WalletWrite: WalletRead { /// There may be restrictions on how far it is possible to rewind. async fn rewind_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error>; } + +/// This trait provides a generalization over shielded output representations. +pub trait ShieldedOutput { + fn index(&self) -> usize; + fn account(&self) -> AccountId; + fn to(&self) -> &PaymentAddress; + fn note(&self) -> &Note; + fn memo(&self) -> Option<&MemoBytes>; + fn is_change(&self) -> Option; + fn nullifier(&self) -> Option; +} + +impl ShieldedOutput for WalletShieldedOutput { + fn index(&self) -> usize { + self.index + } + fn account(&self) -> AccountId { + self.account + } + fn to(&self) -> &PaymentAddress { + &self.to + } + fn note(&self) -> &Note { + &self.note + } + fn memo(&self) -> Option<&MemoBytes> { + None + } + fn is_change(&self) -> Option { + Some(self.is_change) + } + + fn nullifier(&self) -> Option { + Some(self.nf) + } +} + +impl ShieldedOutput for DecryptedOutput { + fn index(&self) -> usize { + self.index + } + fn account(&self) -> AccountId { + self.account + } + fn to(&self) -> &PaymentAddress { + &self.to + } + fn note(&self) -> &Note { + &self.note + } + fn memo(&self) -> Option<&MemoBytes> { + Some(&self.memo) + } + fn is_change(&self) -> Option { + None + } + fn nullifier(&self) -> Option { + None + } +} From 5d5d82202776e52be7899adf224b68f330d96067 Mon Sep 17 00:00:00 2001 From: borngraced Date: Tue, 5 Sep 2023 14:39:52 +0100 Subject: [PATCH 15/42] fix review notes and add doc comments --- zcash_client_sqlite/src/with_async/mod.rs | 24 ++++++++++--------- .../src/with_async/wallet_actions.rs | 2 +- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/zcash_client_sqlite/src/with_async/mod.rs b/zcash_client_sqlite/src/with_async/mod.rs index 3fa590dbb9..a10922dbcd 100644 --- a/zcash_client_sqlite/src/with_async/mod.rs +++ b/zcash_client_sqlite/src/with_async/mod.rs @@ -425,18 +425,20 @@ impl WalletWrite for DataConnS received_tx: &ReceivedTransaction, ) -> Result { self.transactionally(|up| { - let db = up.wallet_db.inner.lock().unwrap(); - let tx_ref = wallet_actions::put_tx_data(&db, received_tx.tx, None)?; - - for output in received_tx.outputs { - if output.outgoing { - wallet_actions::put_sent_note(&db, output, tx_ref)?; - } else { - wallet_actions::put_received_note(&db, output, tx_ref)?; - } - } + // let db = up.wallet_db.inner.lock().unwrap(); + // let tx_ref = wallet_actions::put_tx_data(&db, received_tx.tx, None)?; + // + // for output in received_tx.outputs { + // if output.outgoing { + // wallet_actions::put_sent_note(&db, output, tx_ref)?; + // } else { + // wallet_actions::put_received_note(&db, output, tx_ref)?; + // } + // } + // + // Ok(tx_ref) - Ok(tx_ref) + todo!() }) } diff --git a/zcash_client_sqlite/src/with_async/wallet_actions.rs b/zcash_client_sqlite/src/with_async/wallet_actions.rs index 08a6bffb23..5823abfca1 100644 --- a/zcash_client_sqlite/src/with_async/wallet_actions.rs +++ b/zcash_client_sqlite/src/with_async/wallet_actions.rs @@ -247,7 +247,7 @@ pub fn put_tx_data

( )? .execute(params![ txid, - created_at, + created_at., u32::from(tx.expiry_height), raw_tx ])?; From d3d81c9b7ed7a424c0efc054171f761d3e0a4d1e Mon Sep 17 00:00:00 2001 From: borngraced Date: Tue, 5 Sep 2023 14:40:32 +0100 Subject: [PATCH 16/42] Revert "move ShieldedOutput" This reverts commit 3c21128cd155802355b68606f00e87e09cc41e2c. --- zcash_client_sqlite/src/wallet.rs | 65 +++++++++++++++++- .../src/with_async/wallet_actions.rs | 2 +- zcash_extras/src/lib.rs | 67 +------------------ 3 files changed, 66 insertions(+), 68 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index a983f72af9..c134417da2 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -17,7 +17,7 @@ use zcash_primitives::{ consensus::{self, BlockHeight, NetworkUpgrade}, memo::{Memo, MemoBytes}, merkle_tree::{CommitmentTree, IncrementalWitness}, - sapling::{Node, Nullifier, PaymentAddress}, + sapling::{Node, Note, Nullifier, PaymentAddress}, transaction::{components::Amount, Transaction, TxId}, zip32::ExtendedFullViewingKey, }; @@ -29,16 +29,75 @@ use zcash_client_backend::{ decode_extended_full_viewing_key, decode_payment_address, encode_extended_full_viewing_key, encode_payment_address, }, - wallet::{AccountId, WalletTx}, + wallet::{AccountId, WalletShieldedOutput, WalletTx}, DecryptedOutput, }; -use zcash_extras::ShieldedOutput; use crate::{error::SqliteClientError, DataConnStmtCache, NoteId, WalletDb}; pub mod init; pub mod transact; +/// This trait provides a generalization over shielded output representations. +pub trait ShieldedOutput { + fn index(&self) -> usize; + fn account(&self) -> AccountId; + fn to(&self) -> &PaymentAddress; + fn note(&self) -> &Note; + fn memo(&self) -> Option<&MemoBytes>; + fn is_change(&self) -> Option; + fn nullifier(&self) -> Option; +} + +impl ShieldedOutput for WalletShieldedOutput { + fn index(&self) -> usize { + self.index + } + fn account(&self) -> AccountId { + self.account + } + fn to(&self) -> &PaymentAddress { + &self.to + } + fn note(&self) -> &Note { + &self.note + } + fn memo(&self) -> Option<&MemoBytes> { + None + } + fn is_change(&self) -> Option { + Some(self.is_change) + } + + fn nullifier(&self) -> Option { + Some(self.nf) + } +} + +impl ShieldedOutput for DecryptedOutput { + fn index(&self) -> usize { + self.index + } + fn account(&self) -> AccountId { + self.account + } + fn to(&self) -> &PaymentAddress { + &self.to + } + fn note(&self) -> &Note { + &self.note + } + fn memo(&self) -> Option<&MemoBytes> { + Some(&self.memo) + } + fn is_change(&self) -> Option { + None + } + fn nullifier(&self) -> Option { + None + } +} + /// Returns the address for the account. /// /// # Examples diff --git a/zcash_client_sqlite/src/with_async/wallet_actions.rs b/zcash_client_sqlite/src/with_async/wallet_actions.rs index 5823abfca1..57b32f87b7 100644 --- a/zcash_client_sqlite/src/with_async/wallet_actions.rs +++ b/zcash_client_sqlite/src/with_async/wallet_actions.rs @@ -1,4 +1,5 @@ use crate::error::SqliteClientError; +use crate::wallet::ShieldedOutput; use crate::{NoteId, WalletDb}; use ff::PrimeField; use rusqlite::{params, ToSql}; @@ -7,7 +8,6 @@ use zcash_client_backend::address::RecipientAddress; use zcash_client_backend::encoding::encode_payment_address; use zcash_client_backend::wallet::{AccountId, WalletTx}; use zcash_client_backend::DecryptedOutput; -use zcash_extras::ShieldedOutput; use zcash_primitives::block::BlockHash; use zcash_primitives::consensus; use zcash_primitives::consensus::BlockHeight; diff --git a/zcash_extras/src/lib.rs b/zcash_extras/src/lib.rs index d4739d4cc5..3cb684e1c7 100644 --- a/zcash_extras/src/lib.rs +++ b/zcash_extras/src/lib.rs @@ -5,17 +5,16 @@ use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET; use zcash_client_backend::data_api::PrunedBlock; use zcash_client_backend::data_api::ReceivedTransaction; use zcash_client_backend::data_api::SentTransaction; +use zcash_client_backend::wallet::AccountId; use zcash_client_backend::wallet::SpendableNote; -use zcash_client_backend::wallet::{AccountId, WalletShieldedOutput}; -use zcash_client_backend::DecryptedOutput; use zcash_primitives::block::BlockHash; use zcash_primitives::consensus::BlockHeight; -use zcash_primitives::memo::{Memo, MemoBytes}; +use zcash_primitives::memo::Memo; use zcash_primitives::merkle_tree::CommitmentTree; use zcash_primitives::merkle_tree::IncrementalWitness; +use zcash_primitives::sapling::Node; use zcash_primitives::sapling::Nullifier; use zcash_primitives::sapling::PaymentAddress; -use zcash_primitives::sapling::{Node, Note}; use zcash_primitives::transaction::components::Amount; use zcash_primitives::transaction::TxId; use zcash_primitives::zip32::ExtendedFullViewingKey; @@ -193,63 +192,3 @@ pub trait WalletWrite: WalletRead { /// There may be restrictions on how far it is possible to rewind. async fn rewind_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error>; } - -/// This trait provides a generalization over shielded output representations. -pub trait ShieldedOutput { - fn index(&self) -> usize; - fn account(&self) -> AccountId; - fn to(&self) -> &PaymentAddress; - fn note(&self) -> &Note; - fn memo(&self) -> Option<&MemoBytes>; - fn is_change(&self) -> Option; - fn nullifier(&self) -> Option; -} - -impl ShieldedOutput for WalletShieldedOutput { - fn index(&self) -> usize { - self.index - } - fn account(&self) -> AccountId { - self.account - } - fn to(&self) -> &PaymentAddress { - &self.to - } - fn note(&self) -> &Note { - &self.note - } - fn memo(&self) -> Option<&MemoBytes> { - None - } - fn is_change(&self) -> Option { - Some(self.is_change) - } - - fn nullifier(&self) -> Option { - Some(self.nf) - } -} - -impl ShieldedOutput for DecryptedOutput { - fn index(&self) -> usize { - self.index - } - fn account(&self) -> AccountId { - self.account - } - fn to(&self) -> &PaymentAddress { - &self.to - } - fn note(&self) -> &Note { - &self.note - } - fn memo(&self) -> Option<&MemoBytes> { - Some(&self.memo) - } - fn is_change(&self) -> Option { - None - } - fn nullifier(&self) -> Option { - None - } -} From 8c12241e577bfe38283a0b9acb9d2bcfff30ab71 Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 7 Sep 2023 16:17:09 +0100 Subject: [PATCH 17/42] async init --- zcash_client_sqlite/src/with_async/init.rs | 155 ++++++++++++++++++ zcash_client_sqlite/src/with_async/mod.rs | 25 ++- .../src/with_async/wallet_actions.rs | 2 +- 3 files changed, 168 insertions(+), 14 deletions(-) create mode 100644 zcash_client_sqlite/src/with_async/init.rs diff --git a/zcash_client_sqlite/src/with_async/init.rs b/zcash_client_sqlite/src/with_async/init.rs new file mode 100644 index 0000000000..435a78acf5 --- /dev/null +++ b/zcash_client_sqlite/src/with_async/init.rs @@ -0,0 +1,155 @@ +use crate::address_from_extfvk; +use crate::error::SqliteClientError; +use crate::with_async::WalletDbAsync; +use rusqlite::ToSql; +use zcash_client_backend::encoding::encode_extended_full_viewing_key; +use zcash_primitives::block::BlockHash; +use zcash_primitives::consensus; +use zcash_primitives::consensus::BlockHeight; +use zcash_primitives::zip32::ExtendedFullViewingKey; + +pub fn init_wallet_db

(wdb: &WalletDbAsync

) -> Result<(), rusqlite::Error> { + let wdb = wdb.inner.lock().unwrap(); + wdb.conn.execute( + "CREATE TABLE IF NOT EXISTS accounts ( + account INTEGER PRIMARY KEY, + extfvk TEXT NOT NULL, + address TEXT NOT NULL + )", + [], + )?; + wdb.conn.execute( + "CREATE TABLE IF NOT EXISTS blocks ( + height INTEGER PRIMARY KEY, + hash BLOB NOT NULL, + time INTEGER NOT NULL, + sapling_tree BLOB NOT NULL + )", + [], + )?; + wdb.conn.execute( + "CREATE TABLE IF NOT EXISTS transactions ( + id_tx INTEGER PRIMARY KEY, + txid BLOB NOT NULL UNIQUE, + created TEXT, + block INTEGER, + tx_index INTEGER, + expiry_height INTEGER, + raw BLOB, + FOREIGN KEY (block) REFERENCES blocks(height) + )", + [], + )?; + wdb.conn.execute( + "CREATE TABLE IF NOT EXISTS received_notes ( + id_note INTEGER PRIMARY KEY, + tx INTEGER NOT NULL, + output_index INTEGER NOT NULL, + account INTEGER NOT NULL, + diversifier BLOB NOT NULL, + value INTEGER NOT NULL, + rcm BLOB NOT NULL, + nf BLOB NOT NULL UNIQUE, + is_change INTEGER NOT NULL, + memo BLOB, + spent INTEGER, + FOREIGN KEY (tx) REFERENCES transactions(id_tx), + FOREIGN KEY (account) REFERENCES accounts(account), + FOREIGN KEY (spent) REFERENCES transactions(id_tx), + CONSTRAINT tx_output UNIQUE (tx, output_index) + )", + [], + )?; + wdb.conn.execute( + "CREATE TABLE IF NOT EXISTS sapling_witnesses ( + id_witness INTEGER PRIMARY KEY, + note INTEGER NOT NULL, + block INTEGER NOT NULL, + witness BLOB NOT NULL, + FOREIGN KEY (note) REFERENCES received_notes(id_note), + FOREIGN KEY (block) REFERENCES blocks(height), + CONSTRAINT witness_height UNIQUE (note, block) + )", + [], + )?; + wdb.conn.execute( + "CREATE TABLE IF NOT EXISTS sent_notes ( + id_note INTEGER PRIMARY KEY, + tx INTEGER NOT NULL, + output_index INTEGER NOT NULL, + from_account INTEGER NOT NULL, + address TEXT NOT NULL, + value INTEGER NOT NULL, + memo BLOB, + FOREIGN KEY (tx) REFERENCES transactions(id_tx), + FOREIGN KEY (from_account) REFERENCES accounts(account), + CONSTRAINT tx_output UNIQUE (tx, output_index) + )", + [], + )?; + Ok(()) +} + +pub fn init_accounts_table( + wdb: &WalletDbAsync

, + extfvks: &[ExtendedFullViewingKey], +) -> Result<(), SqliteClientError> { + let wdb = wdb.inner.lock().unwrap(); + + let mut empty_check = wdb.conn.prepare("SELECT * FROM accounts LIMIT 1")?; + if empty_check.exists([])? { + return Err(SqliteClientError::TableNotEmpty); + } + + // Insert accounts atomically + wdb.conn.execute("BEGIN IMMEDIATE", [])?; + for (account, extfvk) in extfvks.iter().enumerate() { + let extfvk_str = encode_extended_full_viewing_key( + wdb.params.hrp_sapling_extended_full_viewing_key(), + extfvk, + ); + + let address_str = address_from_extfvk(&wdb.params, extfvk); + + wdb.conn.execute( + "INSERT INTO accounts (account, extfvk, address) + VALUES (?, ?, ?)", + [ + (account as u32).to_sql()?, + extfvk_str.to_sql()?, + address_str.to_sql()?, + ], + )?; + } + wdb.conn.execute("COMMIT", [])?; + + Ok(()) +} + +pub fn init_blocks_table

( + wdb: &WalletDbAsync

, + height: BlockHeight, + hash: BlockHash, + time: u32, + sapling_tree: &[u8], +) -> Result<(), SqliteClientError> { + let wdb = wdb.inner.lock().unwrap(); + + let mut empty_check = wdb.conn.prepare("SELECT * FROM blocks LIMIT 1")?; + if empty_check.exists([])? { + return Err(SqliteClientError::TableNotEmpty); + } + + wdb.conn.execute( + "INSERT INTO blocks (height, hash, time, sapling_tree) + VALUES (?, ?, ?, ?)", + [ + u32::from(height).to_sql()?, + hash.0.to_sql()?, + time.to_sql()?, + sapling_tree.to_sql()?, + ], + )?; + + Ok(()) +} diff --git a/zcash_client_sqlite/src/with_async/mod.rs b/zcash_client_sqlite/src/with_async/mod.rs index a10922dbcd..e3488f9fd6 100644 --- a/zcash_client_sqlite/src/with_async/mod.rs +++ b/zcash_client_sqlite/src/with_async/mod.rs @@ -1,3 +1,4 @@ +pub mod init; pub mod wallet_actions; use std::collections::HashMap; @@ -425,20 +426,18 @@ impl WalletWrite for DataConnS received_tx: &ReceivedTransaction, ) -> Result { self.transactionally(|up| { - // let db = up.wallet_db.inner.lock().unwrap(); - // let tx_ref = wallet_actions::put_tx_data(&db, received_tx.tx, None)?; - // - // for output in received_tx.outputs { - // if output.outgoing { - // wallet_actions::put_sent_note(&db, output, tx_ref)?; - // } else { - // wallet_actions::put_received_note(&db, output, tx_ref)?; - // } - // } - // - // Ok(tx_ref) + let db = up.wallet_db.inner.lock().unwrap(); + let tx_ref = wallet_actions::put_tx_data(&db, received_tx.tx, None)?; + + for output in received_tx.outputs { + if output.outgoing { + wallet_actions::put_sent_note(&db, output, tx_ref)?; + } else { + wallet_actions::put_received_note(&db, output, tx_ref)?; + } + } - todo!() + Ok(tx_ref) }) } diff --git a/zcash_client_sqlite/src/with_async/wallet_actions.rs b/zcash_client_sqlite/src/with_async/wallet_actions.rs index 57b32f87b7..babb1450a9 100644 --- a/zcash_client_sqlite/src/with_async/wallet_actions.rs +++ b/zcash_client_sqlite/src/with_async/wallet_actions.rs @@ -247,7 +247,7 @@ pub fn put_tx_data

( )? .execute(params![ txid, - created_at., + created_at, u32::from(tx.expiry_height), raw_tx ])?; From e89819f2583a577368d8c72177140232d8c4ee5b Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 7 Sep 2023 18:29:33 +0100 Subject: [PATCH 18/42] move WalletShieldedOutput --- zcash_client_sqlite/src/wallet.rs | 65 +----------------- .../src/with_async/wallet_actions.rs | 2 +- zcash_extras/src/lib.rs | 67 ++++++++++++++++++- 3 files changed, 68 insertions(+), 66 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index c134417da2..a983f72af9 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -17,7 +17,7 @@ use zcash_primitives::{ consensus::{self, BlockHeight, NetworkUpgrade}, memo::{Memo, MemoBytes}, merkle_tree::{CommitmentTree, IncrementalWitness}, - sapling::{Node, Note, Nullifier, PaymentAddress}, + sapling::{Node, Nullifier, PaymentAddress}, transaction::{components::Amount, Transaction, TxId}, zip32::ExtendedFullViewingKey, }; @@ -29,75 +29,16 @@ use zcash_client_backend::{ decode_extended_full_viewing_key, decode_payment_address, encode_extended_full_viewing_key, encode_payment_address, }, - wallet::{AccountId, WalletShieldedOutput, WalletTx}, + wallet::{AccountId, WalletTx}, DecryptedOutput, }; +use zcash_extras::ShieldedOutput; use crate::{error::SqliteClientError, DataConnStmtCache, NoteId, WalletDb}; pub mod init; pub mod transact; -/// This trait provides a generalization over shielded output representations. -pub trait ShieldedOutput { - fn index(&self) -> usize; - fn account(&self) -> AccountId; - fn to(&self) -> &PaymentAddress; - fn note(&self) -> &Note; - fn memo(&self) -> Option<&MemoBytes>; - fn is_change(&self) -> Option; - fn nullifier(&self) -> Option; -} - -impl ShieldedOutput for WalletShieldedOutput { - fn index(&self) -> usize { - self.index - } - fn account(&self) -> AccountId { - self.account - } - fn to(&self) -> &PaymentAddress { - &self.to - } - fn note(&self) -> &Note { - &self.note - } - fn memo(&self) -> Option<&MemoBytes> { - None - } - fn is_change(&self) -> Option { - Some(self.is_change) - } - - fn nullifier(&self) -> Option { - Some(self.nf) - } -} - -impl ShieldedOutput for DecryptedOutput { - fn index(&self) -> usize { - self.index - } - fn account(&self) -> AccountId { - self.account - } - fn to(&self) -> &PaymentAddress { - &self.to - } - fn note(&self) -> &Note { - &self.note - } - fn memo(&self) -> Option<&MemoBytes> { - Some(&self.memo) - } - fn is_change(&self) -> Option { - None - } - fn nullifier(&self) -> Option { - None - } -} - /// Returns the address for the account. /// /// # Examples diff --git a/zcash_client_sqlite/src/with_async/wallet_actions.rs b/zcash_client_sqlite/src/with_async/wallet_actions.rs index babb1450a9..08a6bffb23 100644 --- a/zcash_client_sqlite/src/with_async/wallet_actions.rs +++ b/zcash_client_sqlite/src/with_async/wallet_actions.rs @@ -1,5 +1,4 @@ use crate::error::SqliteClientError; -use crate::wallet::ShieldedOutput; use crate::{NoteId, WalletDb}; use ff::PrimeField; use rusqlite::{params, ToSql}; @@ -8,6 +7,7 @@ use zcash_client_backend::address::RecipientAddress; use zcash_client_backend::encoding::encode_payment_address; use zcash_client_backend::wallet::{AccountId, WalletTx}; use zcash_client_backend::DecryptedOutput; +use zcash_extras::ShieldedOutput; use zcash_primitives::block::BlockHash; use zcash_primitives::consensus; use zcash_primitives::consensus::BlockHeight; diff --git a/zcash_extras/src/lib.rs b/zcash_extras/src/lib.rs index 3cb684e1c7..d4739d4cc5 100644 --- a/zcash_extras/src/lib.rs +++ b/zcash_extras/src/lib.rs @@ -5,16 +5,17 @@ use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET; use zcash_client_backend::data_api::PrunedBlock; use zcash_client_backend::data_api::ReceivedTransaction; use zcash_client_backend::data_api::SentTransaction; -use zcash_client_backend::wallet::AccountId; use zcash_client_backend::wallet::SpendableNote; +use zcash_client_backend::wallet::{AccountId, WalletShieldedOutput}; +use zcash_client_backend::DecryptedOutput; use zcash_primitives::block::BlockHash; use zcash_primitives::consensus::BlockHeight; -use zcash_primitives::memo::Memo; +use zcash_primitives::memo::{Memo, MemoBytes}; use zcash_primitives::merkle_tree::CommitmentTree; use zcash_primitives::merkle_tree::IncrementalWitness; -use zcash_primitives::sapling::Node; use zcash_primitives::sapling::Nullifier; use zcash_primitives::sapling::PaymentAddress; +use zcash_primitives::sapling::{Node, Note}; use zcash_primitives::transaction::components::Amount; use zcash_primitives::transaction::TxId; use zcash_primitives::zip32::ExtendedFullViewingKey; @@ -192,3 +193,63 @@ pub trait WalletWrite: WalletRead { /// There may be restrictions on how far it is possible to rewind. async fn rewind_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error>; } + +/// This trait provides a generalization over shielded output representations. +pub trait ShieldedOutput { + fn index(&self) -> usize; + fn account(&self) -> AccountId; + fn to(&self) -> &PaymentAddress; + fn note(&self) -> &Note; + fn memo(&self) -> Option<&MemoBytes>; + fn is_change(&self) -> Option; + fn nullifier(&self) -> Option; +} + +impl ShieldedOutput for WalletShieldedOutput { + fn index(&self) -> usize { + self.index + } + fn account(&self) -> AccountId { + self.account + } + fn to(&self) -> &PaymentAddress { + &self.to + } + fn note(&self) -> &Note { + &self.note + } + fn memo(&self) -> Option<&MemoBytes> { + None + } + fn is_change(&self) -> Option { + Some(self.is_change) + } + + fn nullifier(&self) -> Option { + Some(self.nf) + } +} + +impl ShieldedOutput for DecryptedOutput { + fn index(&self) -> usize { + self.index + } + fn account(&self) -> AccountId { + self.account + } + fn to(&self) -> &PaymentAddress { + &self.to + } + fn note(&self) -> &Note { + &self.note + } + fn memo(&self) -> Option<&MemoBytes> { + Some(&self.memo) + } + fn is_change(&self) -> Option { + None + } + fn nullifier(&self) -> Option { + None + } +} From 0c2021095812012af1bc01269c012a7e7e5889f7 Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 14 Sep 2023 10:42:07 +0100 Subject: [PATCH 19/42] make fake_compact_block and fake_compact_block_spending public --- zcash_client_sqlite/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index ad3b5a8d1a..432fd5012c 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -609,7 +609,7 @@ mod tests { /// Create a fake CompactBlock at the given height, containing a single output paying /// the given address. Returns the CompactBlock and the nullifier for the new note. - pub(crate) fn fake_compact_block( + pub fn fake_compact_block( height: BlockHeight, prev_hash: BlockHash, extfvk: ExtendedFullViewingKey, @@ -658,7 +658,7 @@ mod tests { /// Create a fake CompactBlock at the given height, spending a single note from the /// given address. - pub(crate) fn fake_compact_block_spending( + pub fn fake_compact_block_spending( height: BlockHeight, prev_hash: BlockHash, (nf, in_value): (Nullifier, Amount), From 3d07af946c8197276e48ecebdf8b4ae480f29792 Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 14 Sep 2023 11:32:21 +0100 Subject: [PATCH 20/42] move fake_compact_block and fake_compact_block_spending public --- zcash_extras/Cargo.toml | 5 ++ zcash_extras/src/lib.rs | 182 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+) diff --git a/zcash_extras/Cargo.toml b/zcash_extras/Cargo.toml index af335c77e2..98a3ab451e 100644 --- a/zcash_extras/Cargo.toml +++ b/zcash_extras/Cargo.toml @@ -7,5 +7,10 @@ edition = "2021" [dependencies] async-trait = "0.1.52" +group = "0.8" +ff = "0.8" +jubjub = "0.5.1" +protobuf = "2.20" +rand_core = "0.5.1" zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } zcash_primitives = { version = "0.5", path = "../zcash_primitives" } diff --git a/zcash_extras/src/lib.rs b/zcash_extras/src/lib.rs index d4739d4cc5..f0f3d8d618 100644 --- a/zcash_extras/src/lib.rs +++ b/zcash_extras/src/lib.rs @@ -253,3 +253,185 @@ impl ShieldedOutput for DecryptedOutput { None } } + +#[cfg(test)] +mod tests { + use ff::PrimeField; + use group::GroupEncoding; + use protobuf::Message; + use rand_core::{OsRng, RngCore}; + + use zcash_client_backend::proto::compact_formats::{ + CompactBlock, CompactOutput, CompactSpend, CompactTx, + }; + + use zcash_primitives::{ + block::BlockHash, + consensus::{BlockHeight, Network, NetworkUpgrade, Parameters}, + memo::MemoBytes, + sapling::{ + note_encryption::sapling_note_encryption, util::generate_random_rseed, Note, Nullifier, + PaymentAddress, + }, + transaction::components::Amount, + zip32::ExtendedFullViewingKey, + }; + + #[cfg(feature = "mainnet")] + pub(crate) fn network() -> Network { + Network::MainNetwork + } + + #[cfg(not(feature = "mainnet"))] + pub(crate) fn network() -> Network { + Network::TestNetwork + } + + #[cfg(feature = "mainnet")] + pub(crate) fn sapling_activation_height() -> BlockHeight { + Network::MainNetwork + .activation_height(NetworkUpgrade::Sapling) + .unwrap() + } + + #[cfg(not(feature = "mainnet"))] + pub(crate) fn sapling_activation_height() -> BlockHeight { + Network::TestNetwork + .activation_height(NetworkUpgrade::Sapling) + .unwrap() + } + + /// Create a fake CompactBlock at the given height, containing a single output paying + /// the given address. Returns the CompactBlock and the nullifier for the new note. + pub fn fake_compact_block( + height: BlockHeight, + prev_hash: BlockHash, + extfvk: ExtendedFullViewingKey, + value: Amount, + ) -> (CompactBlock, Nullifier) { + let to = extfvk.default_address().unwrap().1; + + // Create a fake Note for the account + let mut rng = OsRng; + let rseed = generate_random_rseed(&network(), height, &mut rng); + let note = Note { + g_d: to.diversifier().g_d().unwrap(), + pk_d: *to.pk_d(), + value: value.into(), + rseed, + }; + let encryptor = sapling_note_encryption::<_, Network>( + Some(extfvk.fvk.ovk), + note.clone(), + to, + MemoBytes::empty(), + &mut rng, + ); + let cmu = note.cmu().to_repr().as_ref().to_vec(); + let epk = encryptor.epk().to_bytes().to_vec(); + let enc_ciphertext = encryptor.encrypt_note_plaintext(); + + // Create a fake CompactBlock containing the note + let mut cout = CompactOutput::new(); + cout.set_cmu(cmu); + cout.set_epk(epk); + cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); + let mut ctx = CompactTx::new(); + let mut txid = vec![0; 32]; + rng.fill_bytes(&mut txid); + ctx.set_hash(txid); + ctx.outputs.push(cout); + let mut cb = CompactBlock::new(); + cb.set_height(u64::from(height)); + cb.hash.resize(32, 0); + rng.fill_bytes(&mut cb.hash); + cb.prevHash.extend_from_slice(&prev_hash.0); + cb.vtx.push(ctx); + (cb, note.nf(&extfvk.fvk.vk, 0)) + } + + /// Create a fake CompactBlock at the given height, spending a single note from the + /// given address. + pub fn fake_compact_block_spending( + height: BlockHeight, + prev_hash: BlockHash, + (nf, in_value): (Nullifier, Amount), + extfvk: ExtendedFullViewingKey, + to: PaymentAddress, + value: Amount, + ) -> CompactBlock { + let mut rng = OsRng; + let rseed = generate_random_rseed(&network(), height, &mut rng); + + // Create a fake CompactBlock containing the note + let mut cspend = CompactSpend::new(); + cspend.set_nf(nf.to_vec()); + let mut ctx = CompactTx::new(); + let mut txid = vec![0; 32]; + rng.fill_bytes(&mut txid); + ctx.set_hash(txid); + ctx.spends.push(cspend); + + // Create a fake Note for the payment + ctx.outputs.push({ + let note = Note { + g_d: to.diversifier().g_d().unwrap(), + pk_d: *to.pk_d(), + value: value.into(), + rseed, + }; + let encryptor = sapling_note_encryption::<_, Network>( + Some(extfvk.fvk.ovk), + note.clone(), + to, + MemoBytes::empty(), + &mut rng, + ); + let cmu = note.cmu().to_repr().as_ref().to_vec(); + let epk = encryptor.epk().to_bytes().to_vec(); + let enc_ciphertext = encryptor.encrypt_note_plaintext(); + + let mut cout = CompactOutput::new(); + cout.set_cmu(cmu); + cout.set_epk(epk); + cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); + cout + }); + + // Create a fake Note for the change + ctx.outputs.push({ + let change_addr = extfvk.default_address().unwrap().1; + let rseed = generate_random_rseed(&network(), height, &mut rng); + let note = Note { + g_d: change_addr.diversifier().g_d().unwrap(), + pk_d: *change_addr.pk_d(), + value: (in_value - value).into(), + rseed, + }; + let encryptor = sapling_note_encryption::<_, Network>( + Some(extfvk.fvk.ovk), + note.clone(), + change_addr, + MemoBytes::empty(), + &mut rng, + ); + let cmu = note.cmu().to_repr().as_ref().to_vec(); + let epk = encryptor.epk().to_bytes().to_vec(); + let enc_ciphertext = encryptor.encrypt_note_plaintext(); + + let mut cout = CompactOutput::new(); + cout.set_cmu(cmu); + cout.set_epk(epk); + cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); + cout + }); + + let mut cb = CompactBlock::new(); + cb.set_height(u64::from(height)); + cb.hash.resize(32, 0); + rng.fill_bytes(&mut cb.hash); + cb.prevHash.extend_from_slice(&prev_hash.0); + cb.vtx.push(ctx); + cb + } +} From 9837e1048749ddcdba277c768c5d8ea6fde22dc6 Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 14 Sep 2023 11:41:45 +0100 Subject: [PATCH 21/42] minor changes --- zcash_extras/src/lib.rs | 292 +++++++++++++++++++--------------------- 1 file changed, 141 insertions(+), 151 deletions(-) diff --git a/zcash_extras/src/lib.rs b/zcash_extras/src/lib.rs index f0f3d8d618..4d3d945fbc 100644 --- a/zcash_extras/src/lib.rs +++ b/zcash_extras/src/lib.rs @@ -254,66 +254,117 @@ impl ShieldedOutput for DecryptedOutput { } } -#[cfg(test)] -mod tests { - use ff::PrimeField; - use group::GroupEncoding; - use protobuf::Message; - use rand_core::{OsRng, RngCore}; - - use zcash_client_backend::proto::compact_formats::{ - CompactBlock, CompactOutput, CompactSpend, CompactTx, - }; - - use zcash_primitives::{ - block::BlockHash, - consensus::{BlockHeight, Network, NetworkUpgrade, Parameters}, - memo::MemoBytes, - sapling::{ - note_encryption::sapling_note_encryption, util::generate_random_rseed, Note, Nullifier, - PaymentAddress, - }, - transaction::components::Amount, - zip32::ExtendedFullViewingKey, - }; +use ff::PrimeField; +use group::GroupEncoding; +use protobuf::Message; +use rand_core::{OsRng, RngCore}; + +use zcash_client_backend::proto::compact_formats::{ + CompactBlock, CompactOutput, CompactSpend, CompactTx, +}; + +use zcash_primitives::{ + consensus::{Network, NetworkUpgrade, Parameters}, + sapling::{note_encryption::sapling_note_encryption, util::generate_random_rseed}, +}; + +#[cfg(feature = "mainnet")] +pub(crate) fn network() -> Network { + Network::MainNetwork +} - #[cfg(feature = "mainnet")] - pub(crate) fn network() -> Network { - Network::MainNetwork - } +#[cfg(not(feature = "mainnet"))] +pub(crate) fn network() -> Network { + Network::TestNetwork +} - #[cfg(not(feature = "mainnet"))] - pub(crate) fn network() -> Network { - Network::TestNetwork - } +#[cfg(feature = "mainnet")] +pub(crate) fn sapling_activation_height() -> BlockHeight { + Network::MainNetwork + .activation_height(NetworkUpgrade::Sapling) + .unwrap() +} - #[cfg(feature = "mainnet")] - pub(crate) fn sapling_activation_height() -> BlockHeight { - Network::MainNetwork - .activation_height(NetworkUpgrade::Sapling) - .unwrap() - } +#[cfg(not(feature = "mainnet"))] +pub(crate) fn sapling_activation_height() -> BlockHeight { + Network::TestNetwork + .activation_height(NetworkUpgrade::Sapling) + .unwrap() +} - #[cfg(not(feature = "mainnet"))] - pub(crate) fn sapling_activation_height() -> BlockHeight { - Network::TestNetwork - .activation_height(NetworkUpgrade::Sapling) - .unwrap() - } +/// Create a fake CompactBlock at the given height, containing a single output paying +/// the given address. Returns the CompactBlock and the nullifier for the new note. +pub fn fake_compact_block( + height: BlockHeight, + prev_hash: BlockHash, + extfvk: ExtendedFullViewingKey, + value: Amount, +) -> (CompactBlock, Nullifier) { + let to = extfvk.default_address().unwrap().1; + + // Create a fake Note for the account + let mut rng = OsRng; + let rseed = generate_random_rseed(&network(), height, &mut rng); + let note = Note { + g_d: to.diversifier().g_d().unwrap(), + pk_d: *to.pk_d(), + value: value.into(), + rseed, + }; + let encryptor = sapling_note_encryption::<_, Network>( + Some(extfvk.fvk.ovk), + note.clone(), + to, + MemoBytes::empty(), + &mut rng, + ); + let cmu = note.cmu().to_repr().as_ref().to_vec(); + let epk = encryptor.epk().to_bytes().to_vec(); + let enc_ciphertext = encryptor.encrypt_note_plaintext(); + + // Create a fake CompactBlock containing the note + let mut cout = CompactOutput::new(); + cout.set_cmu(cmu); + cout.set_epk(epk); + cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); + let mut ctx = CompactTx::new(); + let mut txid = vec![0; 32]; + rng.fill_bytes(&mut txid); + ctx.set_hash(txid); + ctx.outputs.push(cout); + let mut cb = CompactBlock::new(); + cb.set_height(u64::from(height)); + cb.hash.resize(32, 0); + rng.fill_bytes(&mut cb.hash); + cb.prevHash.extend_from_slice(&prev_hash.0); + cb.vtx.push(ctx); + (cb, note.nf(&extfvk.fvk.vk, 0)) +} - /// Create a fake CompactBlock at the given height, containing a single output paying - /// the given address. Returns the CompactBlock and the nullifier for the new note. - pub fn fake_compact_block( - height: BlockHeight, - prev_hash: BlockHash, - extfvk: ExtendedFullViewingKey, - value: Amount, - ) -> (CompactBlock, Nullifier) { - let to = extfvk.default_address().unwrap().1; - - // Create a fake Note for the account - let mut rng = OsRng; - let rseed = generate_random_rseed(&network(), height, &mut rng); +/// Create a fake CompactBlock at the given height, spending a single note from the +/// given address. +pub fn fake_compact_block_spending( + height: BlockHeight, + prev_hash: BlockHash, + (nf, in_value): (Nullifier, Amount), + extfvk: ExtendedFullViewingKey, + to: PaymentAddress, + value: Amount, +) -> CompactBlock { + let mut rng = OsRng; + let rseed = generate_random_rseed(&network(), height, &mut rng); + + // Create a fake CompactBlock containing the note + let mut cspend = CompactSpend::new(); + cspend.set_nf(nf.to_vec()); + let mut ctx = CompactTx::new(); + let mut txid = vec![0; 32]; + rng.fill_bytes(&mut txid); + ctx.set_hash(txid); + ctx.spends.push(cspend); + + // Create a fake Note for the payment + ctx.outputs.push({ let note = Note { g_d: to.diversifier().g_d().unwrap(), pk_d: *to.pk_d(), @@ -331,107 +382,46 @@ mod tests { let epk = encryptor.epk().to_bytes().to_vec(); let enc_ciphertext = encryptor.encrypt_note_plaintext(); - // Create a fake CompactBlock containing the note let mut cout = CompactOutput::new(); cout.set_cmu(cmu); cout.set_epk(epk); cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); - let mut ctx = CompactTx::new(); - let mut txid = vec![0; 32]; - rng.fill_bytes(&mut txid); - ctx.set_hash(txid); - ctx.outputs.push(cout); - let mut cb = CompactBlock::new(); - cb.set_height(u64::from(height)); - cb.hash.resize(32, 0); - rng.fill_bytes(&mut cb.hash); - cb.prevHash.extend_from_slice(&prev_hash.0); - cb.vtx.push(ctx); - (cb, note.nf(&extfvk.fvk.vk, 0)) - } + cout + }); - /// Create a fake CompactBlock at the given height, spending a single note from the - /// given address. - pub fn fake_compact_block_spending( - height: BlockHeight, - prev_hash: BlockHash, - (nf, in_value): (Nullifier, Amount), - extfvk: ExtendedFullViewingKey, - to: PaymentAddress, - value: Amount, - ) -> CompactBlock { - let mut rng = OsRng; + // Create a fake Note for the change + ctx.outputs.push({ + let change_addr = extfvk.default_address().unwrap().1; let rseed = generate_random_rseed(&network(), height, &mut rng); + let note = Note { + g_d: change_addr.diversifier().g_d().unwrap(), + pk_d: *change_addr.pk_d(), + value: (in_value - value).into(), + rseed, + }; + let encryptor = sapling_note_encryption::<_, Network>( + Some(extfvk.fvk.ovk), + note.clone(), + change_addr, + MemoBytes::empty(), + &mut rng, + ); + let cmu = note.cmu().to_repr().as_ref().to_vec(); + let epk = encryptor.epk().to_bytes().to_vec(); + let enc_ciphertext = encryptor.encrypt_note_plaintext(); - // Create a fake CompactBlock containing the note - let mut cspend = CompactSpend::new(); - cspend.set_nf(nf.to_vec()); - let mut ctx = CompactTx::new(); - let mut txid = vec![0; 32]; - rng.fill_bytes(&mut txid); - ctx.set_hash(txid); - ctx.spends.push(cspend); - - // Create a fake Note for the payment - ctx.outputs.push({ - let note = Note { - g_d: to.diversifier().g_d().unwrap(), - pk_d: *to.pk_d(), - value: value.into(), - rseed, - }; - let encryptor = sapling_note_encryption::<_, Network>( - Some(extfvk.fvk.ovk), - note.clone(), - to, - MemoBytes::empty(), - &mut rng, - ); - let cmu = note.cmu().to_repr().as_ref().to_vec(); - let epk = encryptor.epk().to_bytes().to_vec(); - let enc_ciphertext = encryptor.encrypt_note_plaintext(); - - let mut cout = CompactOutput::new(); - cout.set_cmu(cmu); - cout.set_epk(epk); - cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); - cout - }); - - // Create a fake Note for the change - ctx.outputs.push({ - let change_addr = extfvk.default_address().unwrap().1; - let rseed = generate_random_rseed(&network(), height, &mut rng); - let note = Note { - g_d: change_addr.diversifier().g_d().unwrap(), - pk_d: *change_addr.pk_d(), - value: (in_value - value).into(), - rseed, - }; - let encryptor = sapling_note_encryption::<_, Network>( - Some(extfvk.fvk.ovk), - note.clone(), - change_addr, - MemoBytes::empty(), - &mut rng, - ); - let cmu = note.cmu().to_repr().as_ref().to_vec(); - let epk = encryptor.epk().to_bytes().to_vec(); - let enc_ciphertext = encryptor.encrypt_note_plaintext(); - - let mut cout = CompactOutput::new(); - cout.set_cmu(cmu); - cout.set_epk(epk); - cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); - cout - }); - - let mut cb = CompactBlock::new(); - cb.set_height(u64::from(height)); - cb.hash.resize(32, 0); - rng.fill_bytes(&mut cb.hash); - cb.prevHash.extend_from_slice(&prev_hash.0); - cb.vtx.push(ctx); - cb - } + let mut cout = CompactOutput::new(); + cout.set_cmu(cmu); + cout.set_epk(epk); + cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); + cout + }); + + let mut cb = CompactBlock::new(); + cb.set_height(u64::from(height)); + cb.hash.resize(32, 0); + rng.fill_bytes(&mut cb.hash); + cb.prevHash.extend_from_slice(&prev_hash.0); + cb.vtx.push(ctx); + cb } From 6ab22ea2da9e57733db462b961bd8e1132976a54 Mon Sep 17 00:00:00 2001 From: borngraced Date: Fri, 15 Sep 2023 11:04:59 +0100 Subject: [PATCH 22/42] copy decrypt_and_store_transaction and create_spend_to_address as async --- zcash_client_sqlite/src/wallet/transact.rs | 4 +- zcash_extras/Cargo.toml | 1 + zcash_extras/src/lib.rs | 2 + zcash_extras/src/wallet.rs | 267 +++++++++++++++++++++ 4 files changed, 272 insertions(+), 2 deletions(-) create mode 100644 zcash_extras/src/wallet.rs diff --git a/zcash_client_sqlite/src/wallet/transact.rs b/zcash_client_sqlite/src/wallet/transact.rs index 5fccb430f3..e76f5eec97 100644 --- a/zcash_client_sqlite/src/wallet/transact.rs +++ b/zcash_client_sqlite/src/wallet/transact.rs @@ -610,7 +610,7 @@ mod tests { .query_row( "SELECT raw FROM transactions WHERE id_tx = ?", - &[tx_row], + [tx_row], |row| row.get(0), ) .unwrap(); @@ -623,7 +623,7 @@ mod tests { .query_row( "SELECT output_index FROM sent_notes WHERE tx = ?", - &[tx_row], + [tx_row], |row| row.get(0), ) .unwrap(); diff --git a/zcash_extras/Cargo.toml b/zcash_extras/Cargo.toml index 98a3ab451e..6ef761c330 100644 --- a/zcash_extras/Cargo.toml +++ b/zcash_extras/Cargo.toml @@ -12,5 +12,6 @@ ff = "0.8" jubjub = "0.5.1" protobuf = "2.20" rand_core = "0.5.1" +time = "0.3" zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } zcash_primitives = { version = "0.5", path = "../zcash_primitives" } diff --git a/zcash_extras/src/lib.rs b/zcash_extras/src/lib.rs index 4d3d945fbc..05abcaec16 100644 --- a/zcash_extras/src/lib.rs +++ b/zcash_extras/src/lib.rs @@ -1,3 +1,5 @@ +pub mod wallet; + use std::cmp; use std::collections::HashMap; use std::fmt::Debug; diff --git a/zcash_extras/src/wallet.rs b/zcash_extras/src/wallet.rs new file mode 100644 index 0000000000..4695bb901e --- /dev/null +++ b/zcash_extras/src/wallet.rs @@ -0,0 +1,267 @@ +//! Functions for scanning the chain and extracting relevant information. +use std::fmt::Debug; + +use zcash_primitives::{ + consensus::{self, BranchId, NetworkUpgrade}, + memo::MemoBytes, + sapling::prover::TxProver, + transaction::{ + builder::Builder, + components::{amount::DEFAULT_FEE, Amount}, + Transaction, + }, + zip32::{ExtendedFullViewingKey, ExtendedSpendingKey}, +}; + +use crate::WalletWrite; +use zcash_client_backend::{ + address::RecipientAddress, + data_api::{error::Error, ReceivedTransaction, SentTransaction}, + decrypt_transaction, + wallet::{AccountId, OvkPolicy}, +}; + +pub const ANCHOR_OFFSET: u32 = 10; + +/// Scans a [`Transaction`] for any information that can be decrypted by the accounts in +/// the wallet, and saves it to the wallet. +pub async fn decrypt_and_store_transaction( + params: &P, + data: &mut D, + tx: &Transaction, +) -> Result<(), E> +where + E: From>, + P: consensus::Parameters, + D: WalletWrite, +{ + // Fetch the ExtendedFullViewingKeys we are tracking + let extfvks = data.get_extended_full_viewing_keys().await?; + + // Height is block height for mined transactions, and the "mempool height" (chain height + 1) + // for mempool transactions. + let height = data + .get_tx_height(tx.txid()) + .await? + .or(data + .block_height_extrema() + .await? + .map(|(_, max_height)| max_height + 1)) + .or_else(|| params.activation_height(NetworkUpgrade::Sapling)) + .ok_or(Error::SaplingNotActive)?; + + let outputs = decrypt_transaction(params, height, tx, &extfvks); + if outputs.is_empty() { + Ok(()) + } else { + data.store_received_tx(&ReceivedTransaction { + tx, + outputs: &outputs, + }) + .await?; + + Ok(()) + } +} + +#[allow(clippy::needless_doctest_main)] +/// Creates a transaction paying the specified address from the given account. +/// +/// Returns the row index of the newly-created transaction in the `transactions` table +/// within the data database. The caller can read the raw transaction bytes from the `raw` +/// column in order to broadcast the transaction to the network. +/// +/// Do not call this multiple times in parallel, or you will generate transactions that +/// double-spend the same notes. +/// +/// # Transaction privacy +/// +/// `ovk_policy` specifies the desired policy for which outgoing viewing key should be +/// able to decrypt the outputs of this transaction. This is primarily relevant to +/// wallet recovery from backup; in particular, [`OvkPolicy::Discard`] will prevent the +/// recipient's address, and the contents of `memo`, from ever being recovered from the +/// block chain. (The total value sent can always be inferred by the sender from the spent +/// notes and received change.) +/// +/// Regardless of the specified policy, `create_spend_to_address` saves `to`, `value`, and +/// `memo` in `db_data`. This can be deleted independently of `ovk_policy`. +/// +/// For details on what transaction information is visible to the holder of a full or +/// outgoing viewing key, refer to [ZIP 310]. +/// +/// [ZIP 310]: https://zips.z.cash/zip-0310 +/// +/// # Examples +/// +/// ``` +/// use tempfile::NamedTempFile; +/// use zcash_primitives::{ +/// consensus::{self, Network}, +/// constants::testnet::COIN_TYPE, +/// transaction::components::Amount +/// }; +/// use zcash_proofs::prover::LocalTxProver; +/// use zcash_client_backend::{ +/// keys::spending_key, +/// data_api::wallet::create_spend_to_address, +/// wallet::{AccountId, OvkPolicy}, +/// }; +/// use zcash_client_sqlite::{ +/// WalletDb, +/// error::SqliteClientError, +/// wallet::init::init_wallet_db, +/// }; +/// +/// # // doctests have a problem with sqlite IO, so we ignore errors +/// # // generated in this example code as it's not really testing anything +/// # fn main() { +/// # test(); +/// # } +/// # +/// # fn test() -> Result<(), SqliteClientError> { +/// let tx_prover = match LocalTxProver::with_default_location() { +/// Some(tx_prover) => tx_prover, +/// None => { +/// panic!("Cannot locate the Zcash parameters. Please run zcash-fetch-params or fetch-params.sh to download the parameters, and then re-run the tests."); +/// } +/// }; +/// +/// let account = AccountId(0); +/// let extsk = spending_key(&[0; 32][..], COIN_TYPE, account.0); +/// let to = extsk.default_address().unwrap().1.into(); +/// +/// let data_file = NamedTempFile::new().unwrap(); +/// let db_read = WalletDb::for_path(data_file, Network::TestNetwork).unwrap(); +/// init_wallet_db(&db_read)?; +/// let mut db = db_read.get_update_ops()?; +/// +/// create_spend_to_address( +/// &mut db, +/// &Network::TestNetwork, +/// tx_prover, +/// account, +/// &extsk, +/// &to, +/// Amount::from_u64(1).unwrap(), +/// None, +/// OvkPolicy::Sender, +/// )?; +/// +/// # Ok(()) +/// # } +/// ``` +#[allow(clippy::too_many_arguments)] +pub async fn create_spend_to_address( + wallet_db: &mut D, + params: &P, + prover: impl TxProver, + account: AccountId, + extsk: &ExtendedSpendingKey, + to: &RecipientAddress, + value: Amount, + memo: Option, + ovk_policy: OvkPolicy, +) -> Result +where + E: From>, + P: consensus::Parameters + Clone, + R: Copy + Debug, + D: WalletWrite, +{ + // Check that the ExtendedSpendingKey we have been given corresponds to the + // ExtendedFullViewingKey for the account we are spending from. + let extfvk = ExtendedFullViewingKey::from(extsk); + if !wallet_db.is_valid_account_extfvk(account, &extfvk).await? { + return Err(E::from(Error::InvalidExtSk(account))); + } + + // Apply the outgoing viewing key policy. + let ovk = match ovk_policy { + OvkPolicy::Sender => Some(extfvk.fvk.ovk), + OvkPolicy::Custom(ovk) => Some(ovk), + OvkPolicy::Discard => None, + }; + + // Target the next block, assuming we are up-to-date. + let (height, anchor_height) = wallet_db + .get_target_and_anchor_heights() + .await + .and_then(|x| x.ok_or_else(|| Error::ScanRequired.into()))?; + + let target_value = value + DEFAULT_FEE; + let spendable_notes = wallet_db + .select_spendable_notes(account, target_value, anchor_height) + .await?; + + // Confirm we were able to select sufficient value + let selected_value = spendable_notes.iter().map(|n| n.note_value).sum(); + if selected_value < target_value { + return Err(E::from(Error::InsufficientBalance( + selected_value, + target_value, + ))); + } + + // Create the transaction + let mut builder = Builder::new(params.clone(), height); + for selected in spendable_notes { + let from = extfvk + .fvk + .vk + .to_payment_address(selected.diversifier) + .unwrap(); //DiversifyHash would have to unexpectedly return the zero point for this to be None + + let note = from + .create_note(selected.note_value.into(), selected.rseed) + .unwrap(); + + let merkle_path = selected.witness.path().expect("the tree is not empty"); + + builder + .add_sapling_spend(extsk.clone(), selected.diversifier, note, merkle_path) + .map_err(Error::Builder)?; + } + + match to { + RecipientAddress::Shielded(to) => { + builder.add_sapling_output(ovk, to.clone(), value, memo.clone()) + } + + RecipientAddress::Transparent(to) => builder.add_transparent_output(&to, value), + } + .map_err(Error::Builder)?; + + let consensus_branch_id = BranchId::for_height(params, height); + let (tx, tx_metadata) = builder + .build(consensus_branch_id, &prover) + .map_err(Error::Builder)?; + + let output_index = match to { + // Sapling outputs are shuffled, so we need to look up where the output ended up. + RecipientAddress::Shielded(_) => match tx_metadata.output_index(0) { + Some(idx) => idx, + None => panic!("Output 0 should exist in the transaction"), + }, + RecipientAddress::Transparent(addr) => { + let script = addr.script(); + tx.vout + .iter() + .enumerate() + .find(|(_, tx_out)| tx_out.script_pubkey == script) + .map(|(index, _)| index) + .expect("we sent to this address") + } + }; + + wallet_db + .store_sent_tx(&SentTransaction { + tx: &tx, + created: time::OffsetDateTime::now_utc(), + output_index, + account, + recipient_address: to, + value, + memo, + }) + .await +} From 8f4b5e17b51534c6b99dbb17d93465e825f1041e Mon Sep 17 00:00:00 2001 From: borngraced Date: Fri, 15 Sep 2023 11:59:25 +0100 Subject: [PATCH 23/42] make db initializations async --- zcash_client_sqlite/src/lib.rs | 2 + zcash_client_sqlite/src/with_async/init.rs | 169 ++++++++++++--------- 2 files changed, 100 insertions(+), 71 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 432fd5012c..3df1c1f681 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -32,6 +32,8 @@ // Catch documentation errors caused by code changes. #![deny(broken_intra_doc_links)] +extern crate core; + use std::collections::HashMap; use std::fmt; use std::path::Path; diff --git a/zcash_client_sqlite/src/with_async/init.rs b/zcash_client_sqlite/src/with_async/init.rs index 435a78acf5..23ad5d8e86 100644 --- a/zcash_client_sqlite/src/with_async/init.rs +++ b/zcash_client_sqlite/src/with_async/init.rs @@ -1,6 +1,6 @@ use crate::address_from_extfvk; use crate::error::SqliteClientError; -use crate::with_async::WalletDbAsync; +use crate::with_async::{async_blocking, WalletDbAsync}; use rusqlite::ToSql; use zcash_client_backend::encoding::encode_extended_full_viewing_key; use zcash_primitives::block::BlockHash; @@ -8,27 +8,35 @@ use zcash_primitives::consensus; use zcash_primitives::consensus::BlockHeight; use zcash_primitives::zip32::ExtendedFullViewingKey; -pub fn init_wallet_db

(wdb: &WalletDbAsync

) -> Result<(), rusqlite::Error> { - let wdb = wdb.inner.lock().unwrap(); - wdb.conn.execute( - "CREATE TABLE IF NOT EXISTS accounts ( +pub async fn init_wallet_db( + wdb: WalletDbAsync

, +) -> Result<(), rusqlite::Error> +where + P: Clone + Send + Sync, +{ + let wdb = wdb.inner.clone(); + async_blocking(move || { + let wdb = wdb.lock().unwrap(); + + wdb.conn.execute( + "CREATE TABLE IF NOT EXISTS accounts ( account INTEGER PRIMARY KEY, extfvk TEXT NOT NULL, address TEXT NOT NULL )", - [], - )?; - wdb.conn.execute( - "CREATE TABLE IF NOT EXISTS blocks ( + [], + )?; + wdb.conn.execute( + "CREATE TABLE IF NOT EXISTS blocks ( height INTEGER PRIMARY KEY, hash BLOB NOT NULL, time INTEGER NOT NULL, sapling_tree BLOB NOT NULL )", - [], - )?; - wdb.conn.execute( - "CREATE TABLE IF NOT EXISTS transactions ( + [], + )?; + wdb.conn.execute( + "CREATE TABLE IF NOT EXISTS transactions ( id_tx INTEGER PRIMARY KEY, txid BLOB NOT NULL UNIQUE, created TEXT, @@ -38,10 +46,10 @@ pub fn init_wallet_db

(wdb: &WalletDbAsync

) -> Result<(), rusqlite::Error> raw BLOB, FOREIGN KEY (block) REFERENCES blocks(height) )", - [], - )?; - wdb.conn.execute( - "CREATE TABLE IF NOT EXISTS received_notes ( + [], + )?; + wdb.conn.execute( + "CREATE TABLE IF NOT EXISTS received_notes ( id_note INTEGER PRIMARY KEY, tx INTEGER NOT NULL, output_index INTEGER NOT NULL, @@ -58,10 +66,10 @@ pub fn init_wallet_db

(wdb: &WalletDbAsync

) -> Result<(), rusqlite::Error> FOREIGN KEY (spent) REFERENCES transactions(id_tx), CONSTRAINT tx_output UNIQUE (tx, output_index) )", - [], - )?; - wdb.conn.execute( - "CREATE TABLE IF NOT EXISTS sapling_witnesses ( + [], + )?; + wdb.conn.execute( + "CREATE TABLE IF NOT EXISTS sapling_witnesses ( id_witness INTEGER PRIMARY KEY, note INTEGER NOT NULL, block INTEGER NOT NULL, @@ -70,10 +78,10 @@ pub fn init_wallet_db

(wdb: &WalletDbAsync

) -> Result<(), rusqlite::Error> FOREIGN KEY (block) REFERENCES blocks(height), CONSTRAINT witness_height UNIQUE (note, block) )", - [], - )?; - wdb.conn.execute( - "CREATE TABLE IF NOT EXISTS sent_notes ( + [], + )?; + wdb.conn.execute( + "CREATE TABLE IF NOT EXISTS sent_notes ( id_note INTEGER PRIMARY KEY, tx INTEGER NOT NULL, output_index INTEGER NOT NULL, @@ -85,71 +93,90 @@ pub fn init_wallet_db

(wdb: &WalletDbAsync

) -> Result<(), rusqlite::Error> FOREIGN KEY (from_account) REFERENCES accounts(account), CONSTRAINT tx_output UNIQUE (tx, output_index) )", - [], - )?; - Ok(()) + [], + )?; + Ok(()) + }) + .await } -pub fn init_accounts_table( +pub async fn init_accounts_table( wdb: &WalletDbAsync

, extfvks: &[ExtendedFullViewingKey], -) -> Result<(), SqliteClientError> { - let wdb = wdb.inner.lock().unwrap(); +) -> Result<(), SqliteClientError> +where + P: Clone + Send + Sync, +{ + let wdb = wdb.inner.clone(); + let extfvks = extfvks.to_vec(); - let mut empty_check = wdb.conn.prepare("SELECT * FROM accounts LIMIT 1")?; - if empty_check.exists([])? { - return Err(SqliteClientError::TableNotEmpty); - } + async_blocking(move || { + let wdb = wdb.lock().unwrap(); - // Insert accounts atomically - wdb.conn.execute("BEGIN IMMEDIATE", [])?; - for (account, extfvk) in extfvks.iter().enumerate() { - let extfvk_str = encode_extended_full_viewing_key( - wdb.params.hrp_sapling_extended_full_viewing_key(), - extfvk, - ); + let mut empty_check = wdb.conn.prepare("SELECT * FROM accounts LIMIT 1")?; + if empty_check.exists([])? { + return Err(SqliteClientError::TableNotEmpty); + } - let address_str = address_from_extfvk(&wdb.params, extfvk); + // Insert accounts atomically + wdb.conn.execute("BEGIN IMMEDIATE", [])?; + for (account, extfvk) in extfvks.iter().enumerate() { + let extfvk_str = encode_extended_full_viewing_key( + wdb.params.hrp_sapling_extended_full_viewing_key(), + extfvk, + ); - wdb.conn.execute( - "INSERT INTO accounts (account, extfvk, address) + let address_str = address_from_extfvk(&wdb.params, extfvk); + + wdb.conn.execute( + "INSERT INTO accounts (account, extfvk, address) VALUES (?, ?, ?)", - [ - (account as u32).to_sql()?, - extfvk_str.to_sql()?, - address_str.to_sql()?, - ], - )?; - } - wdb.conn.execute("COMMIT", [])?; + [ + (account as u32).to_sql()?, + extfvk_str.to_sql()?, + address_str.to_sql()?, + ], + )?; + } + wdb.conn.execute("COMMIT", [])?; - Ok(()) + Ok(()) + }) + .await } -pub fn init_blocks_table

( +pub async fn init_blocks_table( wdb: &WalletDbAsync

, height: BlockHeight, hash: BlockHash, time: u32, sapling_tree: &[u8], -) -> Result<(), SqliteClientError> { - let wdb = wdb.inner.lock().unwrap(); +) -> Result<(), SqliteClientError> +where + P: Clone + Send + Sync, +{ + let wdb = wdb.inner.clone(); + let sapling_tree = sapling_tree.to_vec().clone(); - let mut empty_check = wdb.conn.prepare("SELECT * FROM blocks LIMIT 1")?; - if empty_check.exists([])? { - return Err(SqliteClientError::TableNotEmpty); - } + async_blocking(move || { + let wdb = wdb.lock().unwrap(); + let mut empty_check = wdb.conn.prepare("SELECT * FROM blocks LIMIT 1")?; + if empty_check.exists([])? { + return Err(SqliteClientError::TableNotEmpty); + } - wdb.conn.execute( - "INSERT INTO blocks (height, hash, time, sapling_tree) + wdb.conn.execute( + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (?, ?, ?, ?)", - [ - u32::from(height).to_sql()?, - hash.0.to_sql()?, - time.to_sql()?, - sapling_tree.to_sql()?, - ], - )?; + [ + u32::from(height).to_sql()?, + hash.0.to_sql()?, + time.to_sql()?, + sapling_tree.as_slice().to_sql()?, + ], + )?; - Ok(()) + Ok(()) + }) + .await } From 809fc706c63a1a417570298bdee6da0bb31d5ea2 Mon Sep 17 00:00:00 2001 From: borngraced Date: Mon, 18 Sep 2023 11:42:30 +0100 Subject: [PATCH 24/42] minor changes --- zcash_client_sqlite/src/{with_async => for_async}/init.rs | 2 +- zcash_client_sqlite/src/{with_async => for_async}/mod.rs | 0 .../src/{with_async => for_async}/wallet_actions.rs | 0 zcash_primitives/Cargo.toml | 4 ++-- 4 files changed, 3 insertions(+), 3 deletions(-) rename zcash_client_sqlite/src/{with_async => for_async}/init.rs (99%) rename zcash_client_sqlite/src/{with_async => for_async}/mod.rs (100%) rename zcash_client_sqlite/src/{with_async => for_async}/wallet_actions.rs (100%) diff --git a/zcash_client_sqlite/src/with_async/init.rs b/zcash_client_sqlite/src/for_async/init.rs similarity index 99% rename from zcash_client_sqlite/src/with_async/init.rs rename to zcash_client_sqlite/src/for_async/init.rs index 23ad5d8e86..ff5366c37b 100644 --- a/zcash_client_sqlite/src/with_async/init.rs +++ b/zcash_client_sqlite/src/for_async/init.rs @@ -9,7 +9,7 @@ use zcash_primitives::consensus::BlockHeight; use zcash_primitives::zip32::ExtendedFullViewingKey; pub async fn init_wallet_db( - wdb: WalletDbAsync

, + wdb: &WalletDbAsync

, ) -> Result<(), rusqlite::Error> where P: Clone + Send + Sync, diff --git a/zcash_client_sqlite/src/with_async/mod.rs b/zcash_client_sqlite/src/for_async/mod.rs similarity index 100% rename from zcash_client_sqlite/src/with_async/mod.rs rename to zcash_client_sqlite/src/for_async/mod.rs diff --git a/zcash_client_sqlite/src/with_async/wallet_actions.rs b/zcash_client_sqlite/src/for_async/wallet_actions.rs similarity index 100% rename from zcash_client_sqlite/src/with_async/wallet_actions.rs rename to zcash_client_sqlite/src/for_async/wallet_actions.rs diff --git a/zcash_primitives/Cargo.toml b/zcash_primitives/Cargo.toml index 5d90d3e51a..de8815b468 100644 --- a/zcash_primitives/Cargo.toml +++ b/zcash_primitives/Cargo.toml @@ -16,7 +16,7 @@ edition = "2018" all-features = true [dependencies] -aes = "0.8" +aes = "0.7" bitvec = "0.18" blake2b_simd = "0.5" blake2s_simd = "0.5" @@ -25,7 +25,7 @@ byteorder = "1" crypto_api_chachapoly = "0.4" equihash = { version = "0.1", path = "../components/equihash" } ff = "0.8" -fpe = "0.6" +fpe = "0.5" group = "0.8" hex = "0.4" jubjub = "0.5.1" From 985d47dba0037754a7afcfccbb4de38cb487aaf6 Mon Sep 17 00:00:00 2001 From: borngraced Date: Mon, 18 Sep 2023 11:46:12 +0100 Subject: [PATCH 25/42] rename module --- zcash_client_sqlite/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 3df1c1f681..88c23811a7 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -63,8 +63,8 @@ use crate::error::SqliteClientError; pub mod chain; pub mod error; +pub mod for_async; pub mod wallet; -pub mod with_async; /// A newtype wrapper for sqlite primary key values for the notes /// table. From becd9d02fd0b945766db9a10e55e5303641cdfbd Mon Sep 17 00:00:00 2001 From: borngraced Date: Mon, 18 Sep 2023 11:49:18 +0100 Subject: [PATCH 26/42] remove unused --- zcash_client_sqlite/src/for_async/init.rs | 2 +- zcash_extras/src/lib.rs | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/zcash_client_sqlite/src/for_async/init.rs b/zcash_client_sqlite/src/for_async/init.rs index ff5366c37b..a898f74906 100644 --- a/zcash_client_sqlite/src/for_async/init.rs +++ b/zcash_client_sqlite/src/for_async/init.rs @@ -1,6 +1,6 @@ use crate::address_from_extfvk; use crate::error::SqliteClientError; -use crate::with_async::{async_blocking, WalletDbAsync}; +use crate::for_async::{async_blocking, WalletDbAsync}; use rusqlite::ToSql; use zcash_client_backend::encoding::encode_extended_full_viewing_key; use zcash_primitives::block::BlockHash; diff --git a/zcash_extras/src/lib.rs b/zcash_extras/src/lib.rs index 05abcaec16..9eb212b318 100644 --- a/zcash_extras/src/lib.rs +++ b/zcash_extras/src/lib.rs @@ -258,7 +258,6 @@ impl ShieldedOutput for DecryptedOutput { use ff::PrimeField; use group::GroupEncoding; -use protobuf::Message; use rand_core::{OsRng, RngCore}; use zcash_client_backend::proto::compact_formats::{ @@ -266,7 +265,7 @@ use zcash_client_backend::proto::compact_formats::{ }; use zcash_primitives::{ - consensus::{Network, NetworkUpgrade, Parameters}, + consensus::Network, sapling::{note_encryption::sapling_note_encryption, util::generate_random_rseed}, }; @@ -287,13 +286,6 @@ pub(crate) fn sapling_activation_height() -> BlockHeight { .unwrap() } -#[cfg(not(feature = "mainnet"))] -pub(crate) fn sapling_activation_height() -> BlockHeight { - Network::TestNetwork - .activation_height(NetworkUpgrade::Sapling) - .unwrap() -} - /// Create a fake CompactBlock at the given height, containing a single output paying /// the given address. Returns the CompactBlock and the nullifier for the new note. pub fn fake_compact_block( From ef1389821dd6ccf1ed66ad2946a0f915b39549f4 Mon Sep 17 00:00:00 2001 From: borngraced Date: Tue, 19 Sep 2023 21:52:14 +0100 Subject: [PATCH 27/42] make LocalTxProver field pub --- zcash_proofs/src/prover.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zcash_proofs/src/prover.rs b/zcash_proofs/src/prover.rs index a6a588cf86..caf81ddc73 100644 --- a/zcash_proofs/src/prover.rs +++ b/zcash_proofs/src/prover.rs @@ -21,9 +21,9 @@ use crate::{default_params_folder, SAPLING_OUTPUT_NAME, SAPLING_SPEND_NAME}; /// An implementation of [`TxProver`] using Sapling Spend and Output parameters from /// locally-accessible paths. pub struct LocalTxProver { - spend_params: Parameters, - spend_vk: PreparedVerifyingKey, - output_params: Parameters, + pub spend_params: Parameters, + pub spend_vk: PreparedVerifyingKey, + pub output_params: Parameters, } impl LocalTxProver { From 1901f09019dfdbb02d7c770f7958d14abd03f7b8 Mon Sep 17 00:00:00 2001 From: borngraced Date: Tue, 26 Sep 2023 00:33:48 +0100 Subject: [PATCH 28/42] move NoteId structure --- zcash_client_sqlite/src/lib.rs | 35 +++++++++++++-------------- zcash_extras/src/lib.rs | 43 +++++++++++++++++++++------------- zcash_extras/src/wallet.rs | 3 ++- 3 files changed, 47 insertions(+), 34 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 88c23811a7..40b7fd4fa6 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -35,7 +35,7 @@ extern crate core; use std::collections::HashMap; -use std::fmt; +//use std::fmt; use std::path::Path; use rusqlite::{Connection, Statement}; @@ -58,6 +58,7 @@ use zcash_client_backend::{ proto::compact_formats::CompactBlock, wallet::{AccountId, SpendableNote}, }; +use zcash_extras::NoteId; use crate::error::SqliteClientError; @@ -66,22 +67,22 @@ pub mod error; pub mod for_async; pub mod wallet; -/// A newtype wrapper for sqlite primary key values for the notes -/// table. -#[derive(Debug, Copy, Clone)] -pub enum NoteId { - SentNoteId(i64), - ReceivedNoteId(i64), -} - -impl fmt::Display for NoteId { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - NoteId::SentNoteId(id) => write!(f, "Sent Note {}", id), - NoteId::ReceivedNoteId(id) => write!(f, "Received Note {}", id), - } - } -} +///// A newtype wrapper for sqlite primary key values for the notes +///// table. +//#[derive(Debug, Copy, Clone)] +//pub enum NoteId { +// SentNoteId(i64), +// ReceivedNoteId(i64), +//} +// +//impl fmt::Display for NoteId { +// fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { +// match self { +// NoteId::SentNoteId(id) => write!(f, "Sent Note {}", id), +// NoteId::ReceivedNoteId(id) => write!(f, "Received Note {}", id), +// } +// } +//} /// A wrapper for the SQLite connection to the wallet database. pub struct WalletDb

{ diff --git a/zcash_extras/src/lib.rs b/zcash_extras/src/lib.rs index 9eb212b318..3470697818 100644 --- a/zcash_extras/src/lib.rs +++ b/zcash_extras/src/lib.rs @@ -1,12 +1,18 @@ pub mod wallet; -use std::cmp; +use ff::PrimeField; +use group::GroupEncoding; +use rand_core::{OsRng, RngCore}; use std::collections::HashMap; use std::fmt::Debug; +use std::{cmp, fmt}; use zcash_client_backend::data_api::wallet::ANCHOR_OFFSET; use zcash_client_backend::data_api::PrunedBlock; use zcash_client_backend::data_api::ReceivedTransaction; use zcash_client_backend::data_api::SentTransaction; +use zcash_client_backend::proto::compact_formats::{ + CompactBlock, CompactOutput, CompactSpend, CompactTx, +}; use zcash_client_backend::wallet::SpendableNote; use zcash_client_backend::wallet::{AccountId, WalletShieldedOutput}; use zcash_client_backend::DecryptedOutput; @@ -15,12 +21,14 @@ use zcash_primitives::consensus::BlockHeight; use zcash_primitives::memo::{Memo, MemoBytes}; use zcash_primitives::merkle_tree::CommitmentTree; use zcash_primitives::merkle_tree::IncrementalWitness; -use zcash_primitives::sapling::Nullifier; -use zcash_primitives::sapling::PaymentAddress; -use zcash_primitives::sapling::{Node, Note}; +use zcash_primitives::sapling::{Node, Note, Nullifier, PaymentAddress}; use zcash_primitives::transaction::components::Amount; use zcash_primitives::transaction::TxId; use zcash_primitives::zip32::ExtendedFullViewingKey; +use zcash_primitives::{ + consensus::Network, + sapling::{note_encryption::sapling_note_encryption, util::generate_random_rseed}, +}; #[async_trait::async_trait] pub trait WalletRead: Send + Sync + 'static { @@ -256,19 +264,22 @@ impl ShieldedOutput for DecryptedOutput { } } -use ff::PrimeField; -use group::GroupEncoding; -use rand_core::{OsRng, RngCore}; - -use zcash_client_backend::proto::compact_formats::{ - CompactBlock, CompactOutput, CompactSpend, CompactTx, -}; - -use zcash_primitives::{ - consensus::Network, - sapling::{note_encryption::sapling_note_encryption, util::generate_random_rseed}, -}; +/// A newtype wrapper for sqlite primary key values for the notes +/// table. +#[derive(Debug, Copy, Clone)] +pub enum NoteId { + SentNoteId(i64), + ReceivedNoteId(i64), +} +impl fmt::Display for NoteId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + NoteId::SentNoteId(id) => write!(f, "Sent Note {}", id), + NoteId::ReceivedNoteId(id) => write!(f, "Received Note {}", id), + } + } +} #[cfg(feature = "mainnet")] pub(crate) fn network() -> Network { Network::MainNetwork diff --git a/zcash_extras/src/wallet.rs b/zcash_extras/src/wallet.rs index 4695bb901e..0b14fb4cb0 100644 --- a/zcash_extras/src/wallet.rs +++ b/zcash_extras/src/wallet.rs @@ -1,5 +1,5 @@ //! Functions for scanning the chain and extracting relevant information. -use std::fmt::Debug; +use std::fmt::{Debug, Display}; use zcash_primitives::{ consensus::{self, BranchId, NetworkUpgrade}, @@ -163,6 +163,7 @@ pub async fn create_spend_to_address( ovk_policy: OvkPolicy, ) -> Result where + N: Display, E: From>, P: consensus::Parameters + Clone, R: Copy + Debug, From bde9aab497d39950ec066a472afd5c061f93d495 Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 5 Oct 2023 14:32:36 +0100 Subject: [PATCH 29/42] fix review notes --- zcash_client_sqlite/Cargo.toml | 8 ++++---- zcash_extras/Cargo.toml | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 7779855212..536af908ed 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -17,15 +17,15 @@ async-trait = "0.1.52" bech32 = "0.9.1" bs58 = { version = "0.4", features = ["check"] } ff = "0.8" -futures = { version = "0.3", package = "futures", features = ["compat", "async-await"] } +futures = { version = "0.3" } group = "0.8" jubjub = "0.5.1" protobuf = "2.20" rand_core = "0.5.1" -rusqlite = { version = "0.28", features = ["bundled", "time"] } +rusqlite = { version = "0.28", features = ["time"] } libsqlite3-sys= { version = "0.25.2", features = ["bundled"] } -time = "0.3" -tokio = { version = "1.20", features = ["full"] } +time = { version = "0.3", features = ["wasm-bindgen"]} +tokio = { version = "1.20", features = ["rt"] } zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } zcash_extras = { version = "0.1", path = "../zcash_extras" } zcash_primitives = { version = "0.5", path = "../zcash_primitives" } diff --git a/zcash_extras/Cargo.toml b/zcash_extras/Cargo.toml index 6ef761c330..4e37f2e8fd 100644 --- a/zcash_extras/Cargo.toml +++ b/zcash_extras/Cargo.toml @@ -3,8 +3,6 @@ name = "zcash_extras" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] async-trait = "0.1.52" group = "0.8" @@ -12,6 +10,6 @@ ff = "0.8" jubjub = "0.5.1" protobuf = "2.20" rand_core = "0.5.1" -time = "0.3" +time = { version = "0.3", features = ["wasm-bindgen"]} zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } zcash_primitives = { version = "0.5", path = "../zcash_primitives" } From 37623e3b4c9fa6b2ebe712f8f9b0f90044e5dd01 Mon Sep 17 00:00:00 2001 From: borngraced Date: Fri, 6 Oct 2023 01:18:20 +0100 Subject: [PATCH 30/42] update time crate --- zcash_client_sqlite/Cargo.toml | 2 +- zcash_extras/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 536af908ed..de61c65dae 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -24,7 +24,7 @@ protobuf = "2.20" rand_core = "0.5.1" rusqlite = { version = "0.28", features = ["time"] } libsqlite3-sys= { version = "0.25.2", features = ["bundled"] } -time = { version = "0.3", features = ["wasm-bindgen"]} +time = { version = "0.3.20", features = ["wasm-bindgen"]} tokio = { version = "1.20", features = ["rt"] } zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } zcash_extras = { version = "0.1", path = "../zcash_extras" } diff --git a/zcash_extras/Cargo.toml b/zcash_extras/Cargo.toml index 4e37f2e8fd..786dedda71 100644 --- a/zcash_extras/Cargo.toml +++ b/zcash_extras/Cargo.toml @@ -10,6 +10,6 @@ ff = "0.8" jubjub = "0.5.1" protobuf = "2.20" rand_core = "0.5.1" -time = { version = "0.3", features = ["wasm-bindgen"]} +time = { version = "0.3.20", features = ["std", "wasm-bindgen"]} zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } zcash_primitives = { version = "0.5", path = "../zcash_primitives" } From 496d255284ead0998b3d57d046361f2a64e28420 Mon Sep 17 00:00:00 2001 From: borngraced Date: Wed, 11 Oct 2023 13:52:18 +0100 Subject: [PATCH 31/42] fix review notes --- zcash_client_sqlite/src/for_async/init.rs | 132 +--------------------- zcash_extras/Cargo.toml | 7 +- 2 files changed, 10 insertions(+), 129 deletions(-) diff --git a/zcash_client_sqlite/src/for_async/init.rs b/zcash_client_sqlite/src/for_async/init.rs index a898f74906..91d9a0339d 100644 --- a/zcash_client_sqlite/src/for_async/init.rs +++ b/zcash_client_sqlite/src/for_async/init.rs @@ -1,8 +1,6 @@ -use crate::address_from_extfvk; use crate::error::SqliteClientError; use crate::for_async::{async_blocking, WalletDbAsync}; -use rusqlite::ToSql; -use zcash_client_backend::encoding::encode_extended_full_viewing_key; +use crate::wallet; use zcash_primitives::block::BlockHash; use zcash_primitives::consensus; use zcash_primitives::consensus::BlockHeight; @@ -17,85 +15,7 @@ where let wdb = wdb.inner.clone(); async_blocking(move || { let wdb = wdb.lock().unwrap(); - - wdb.conn.execute( - "CREATE TABLE IF NOT EXISTS accounts ( - account INTEGER PRIMARY KEY, - extfvk TEXT NOT NULL, - address TEXT NOT NULL - )", - [], - )?; - wdb.conn.execute( - "CREATE TABLE IF NOT EXISTS blocks ( - height INTEGER PRIMARY KEY, - hash BLOB NOT NULL, - time INTEGER NOT NULL, - sapling_tree BLOB NOT NULL - )", - [], - )?; - wdb.conn.execute( - "CREATE TABLE IF NOT EXISTS transactions ( - id_tx INTEGER PRIMARY KEY, - txid BLOB NOT NULL UNIQUE, - created TEXT, - block INTEGER, - tx_index INTEGER, - expiry_height INTEGER, - raw BLOB, - FOREIGN KEY (block) REFERENCES blocks(height) - )", - [], - )?; - wdb.conn.execute( - "CREATE TABLE IF NOT EXISTS received_notes ( - id_note INTEGER PRIMARY KEY, - tx INTEGER NOT NULL, - output_index INTEGER NOT NULL, - account INTEGER NOT NULL, - diversifier BLOB NOT NULL, - value INTEGER NOT NULL, - rcm BLOB NOT NULL, - nf BLOB NOT NULL UNIQUE, - is_change INTEGER NOT NULL, - memo BLOB, - spent INTEGER, - FOREIGN KEY (tx) REFERENCES transactions(id_tx), - FOREIGN KEY (account) REFERENCES accounts(account), - FOREIGN KEY (spent) REFERENCES transactions(id_tx), - CONSTRAINT tx_output UNIQUE (tx, output_index) - )", - [], - )?; - wdb.conn.execute( - "CREATE TABLE IF NOT EXISTS sapling_witnesses ( - id_witness INTEGER PRIMARY KEY, - note INTEGER NOT NULL, - block INTEGER NOT NULL, - witness BLOB NOT NULL, - FOREIGN KEY (note) REFERENCES received_notes(id_note), - FOREIGN KEY (block) REFERENCES blocks(height), - CONSTRAINT witness_height UNIQUE (note, block) - )", - [], - )?; - wdb.conn.execute( - "CREATE TABLE IF NOT EXISTS sent_notes ( - id_note INTEGER PRIMARY KEY, - tx INTEGER NOT NULL, - output_index INTEGER NOT NULL, - from_account INTEGER NOT NULL, - address TEXT NOT NULL, - value INTEGER NOT NULL, - memo BLOB, - FOREIGN KEY (tx) REFERENCES transactions(id_tx), - FOREIGN KEY (from_account) REFERENCES accounts(account), - CONSTRAINT tx_output UNIQUE (tx, output_index) - )", - [], - )?; - Ok(()) + wallet::init::init_wallet_db(&wdb) }) .await } @@ -112,35 +32,7 @@ where async_blocking(move || { let wdb = wdb.lock().unwrap(); - - let mut empty_check = wdb.conn.prepare("SELECT * FROM accounts LIMIT 1")?; - if empty_check.exists([])? { - return Err(SqliteClientError::TableNotEmpty); - } - - // Insert accounts atomically - wdb.conn.execute("BEGIN IMMEDIATE", [])?; - for (account, extfvk) in extfvks.iter().enumerate() { - let extfvk_str = encode_extended_full_viewing_key( - wdb.params.hrp_sapling_extended_full_viewing_key(), - extfvk, - ); - - let address_str = address_from_extfvk(&wdb.params, extfvk); - - wdb.conn.execute( - "INSERT INTO accounts (account, extfvk, address) - VALUES (?, ?, ?)", - [ - (account as u32).to_sql()?, - extfvk_str.to_sql()?, - address_str.to_sql()?, - ], - )?; - } - wdb.conn.execute("COMMIT", [])?; - - Ok(()) + wallet::init::init_accounts_table(&wdb, &extfvks) }) .await } @@ -160,23 +52,7 @@ where async_blocking(move || { let wdb = wdb.lock().unwrap(); - let mut empty_check = wdb.conn.prepare("SELECT * FROM blocks LIMIT 1")?; - if empty_check.exists([])? { - return Err(SqliteClientError::TableNotEmpty); - } - - wdb.conn.execute( - "INSERT INTO blocks (height, hash, time, sapling_tree) - VALUES (?, ?, ?, ?)", - [ - u32::from(height).to_sql()?, - hash.0.to_sql()?, - time.to_sql()?, - sapling_tree.as_slice().to_sql()?, - ], - )?; - - Ok(()) + wallet::init::init_blocks_table(&wdb, height, hash, time, &sapling_tree) }) .await } diff --git a/zcash_extras/Cargo.toml b/zcash_extras/Cargo.toml index 786dedda71..bc4b95cb8c 100644 --- a/zcash_extras/Cargo.toml +++ b/zcash_extras/Cargo.toml @@ -10,6 +10,11 @@ ff = "0.8" jubjub = "0.5.1" protobuf = "2.20" rand_core = "0.5.1" -time = { version = "0.3.20", features = ["std", "wasm-bindgen"]} zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } zcash_primitives = { version = "0.5", path = "../zcash_primitives" } + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +time = "0.3.20" + +[target.'cfg(target_arch = "wasm32")'.dependencies] +time = { version = "0.3.20", features = ["wasm-bindgen"]} From af14c33498ffad4d0cf19e1eae12099f0594e417 Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 12 Oct 2023 00:15:23 +0100 Subject: [PATCH 32/42] remove redundancy --- zcash_client_backend/Cargo.toml | 8 +- zcash_client_backend/src/data_api.rs | 2 +- zcash_client_sqlite/Cargo.toml | 7 +- .../src/for_async/wallet_actions.rs | 290 +++--------------- zcash_extras/src/lib.rs | 2 +- 5 files changed, 50 insertions(+), 259 deletions(-) diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 8eb7aab3ff..b54470fb3e 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -29,10 +29,16 @@ proptest = { version = "0.10.1", optional = true } protobuf = "2.20" rand_core = "0.5.1" subtle = "2.2.3" -time = "0.3" zcash_note_encryption = { version = "0.0", path = "../components/zcash_note_encryption" } zcash_primitives = { version = "0.5", path = "../zcash_primitives" } +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +time = "0.3.20" + +[target.'cfg(target_arch = "wasm32")'.dependencies] +time = { version = "0.3.20", features = ["wasm-bindgen"]} + + [build-dependencies] protobuf-codegen-pure = "2.20" diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index ff87bde87e..0eef4818c8 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -34,7 +34,7 @@ pub mod wallet; /// abstracted away from any particular data storage substrate. pub trait WalletRead { /// The type of errors produced by a wallet backend. - type Error; + type Error: ToString; /// Backend-specific note identifier. /// diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index de61c65dae..9f2d2874f5 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -24,12 +24,17 @@ protobuf = "2.20" rand_core = "0.5.1" rusqlite = { version = "0.28", features = ["time"] } libsqlite3-sys= { version = "0.25.2", features = ["bundled"] } -time = { version = "0.3.20", features = ["wasm-bindgen"]} tokio = { version = "1.20", features = ["rt"] } zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } zcash_extras = { version = "0.1", path = "../zcash_extras" } zcash_primitives = { version = "0.5", path = "../zcash_primitives" } +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +time = "0.3.20" + +[target.'cfg(target_arch = "wasm32")'.dependencies] +time = { version = "0.3.20", features = ["wasm-bindgen"]} + [dev-dependencies] rand_core = "0.5.1" diff --git a/zcash_client_sqlite/src/for_async/wallet_actions.rs b/zcash_client_sqlite/src/for_async/wallet_actions.rs index 08a6bffb23..b1f1caab8f 100644 --- a/zcash_client_sqlite/src/for_async/wallet_actions.rs +++ b/zcash_client_sqlite/src/for_async/wallet_actions.rs @@ -1,10 +1,8 @@ use crate::error::SqliteClientError; -use crate::{NoteId, WalletDb}; -use ff::PrimeField; -use rusqlite::{params, ToSql}; +use crate::{wallet, NoteId, WalletDb}; +use rusqlite::params; use std::sync::MutexGuard; use zcash_client_backend::address::RecipientAddress; -use zcash_client_backend::encoding::encode_payment_address; use zcash_client_backend::wallet::{AccountId, WalletTx}; use zcash_client_backend::DecryptedOutput; use zcash_extras::ShieldedOutput; @@ -17,7 +15,6 @@ use zcash_primitives::sapling::{Node, Nullifier}; use zcash_primitives::transaction::components::Amount; use zcash_primitives::transaction::Transaction; -/// Inserts information about a scanned block into the database. pub fn insert_block

( db: &MutexGuard>, block_height: BlockHeight, @@ -25,296 +22,86 @@ pub fn insert_block

( block_time: u32, commitment_tree: &CommitmentTree, ) -> Result<(), SqliteClientError> { - let mut encoded_tree = Vec::new(); - - commitment_tree.write(&mut encoded_tree).unwrap(); - db.conn - .prepare( - "INSERT INTO blocks (height, hash, time, sapling_tree) - VALUES (?, ?, ?, ?)", - )? - .execute(params![ - u32::from(block_height), - &block_hash.0[..], - block_time, - encoded_tree - ])?; - - Ok(()) + let mut update_ops = db.get_update_ops()?; + wallet::insert_block( + &mut update_ops, + block_height, + block_hash, + block_time, + commitment_tree, + ) } -/// Inserts information about a mined transaction that was observed to -/// contain a note related to this wallet into the database. pub fn put_tx_meta( db: &MutexGuard>, tx: &WalletTx, height: BlockHeight, ) -> Result { - let txid = tx.txid.0.to_vec(); - if db - .conn - .prepare( - "UPDATE transactions - SET block = ?, tx_index = ? WHERE txid = ?", - )? - .execute(params![u32::from(height), (tx.index as i64), txid])? - == 0 - { - // It isn't there, so insert our transaction into the database. - db.conn - .prepare( - "INSERT INTO transactions (txid, block, tx_index) - VALUES (?, ?, ?)", - )? - .execute(params![txid, u32::from(height), (tx.index as i64),])?; - - Ok(db.conn.last_insert_rowid()) - } else { - // It was there, so grab its row number. - db.conn - .prepare("SELECT id_tx FROM transactions WHERE txid = ?")? - .query_row([txid], |row| row.get(0)) - .map_err(SqliteClientError::from) - } + let mut update_ops = db.get_update_ops()?; + wallet::put_tx_meta(&mut update_ops, tx, height) } -/// Marks a given nullifier as having been revealed in the construction -/// of the specified transaction. -/// -/// Marking a note spent in this fashion does NOT imply that the -/// spending transaction has been mined. pub fn mark_spent

( db: &MutexGuard>, tx_ref: i64, nf: &Nullifier, ) -> Result<(), SqliteClientError> { - db.conn - .prepare("UPDATE received_notes SET spent = ? WHERE nf = ?")? - .execute([tx_ref.to_sql()?, nf.0.to_sql()?])?; - Ok(()) + wallet::mark_spent(&mut db.get_update_ops()?, tx_ref, nf) } -/// Records the specified shielded output as having been received. -// Assumptions: -// - A transaction will not contain more than 2^63 shielded outputs. -// - A note value will never exceed 2^63 zatoshis. pub fn put_received_note( db: &MutexGuard>, output: &T, tx_ref: i64, ) -> Result { - let rcm = output.note().rcm().to_repr(); - let account = output.account().0 as i64; - let diversifier = output.to().diversifier().0.to_vec(); - let value = output.note().value as i64; - let rcm = rcm.as_ref(); - let memo = output.memo().map(|m| m.as_slice()); - let is_change = output.is_change(); - let tx = tx_ref; - let output_index = output.index() as i64; - let nf_bytes = output.nullifier().map(|nf| nf.0.to_vec()); - - let sql_args: &[(&str, &dyn ToSql)] = &[ - (":account", &account), - (":diversifier", &diversifier), - (":value", &value), - (":rcm", &rcm), - (":nf", &nf_bytes), - (":memo", &memo), - (":is_change", &is_change), - (":tx", &tx), - (":output_index", &output_index), - ]; - - // First try updating an existing received note into the database. - if db - .conn - .prepare( - "UPDATE received_notes - SET account = :account, - diversifier = :diversifier, - value = :value, - rcm = :rcm, - nf = IFNULL(:nf, nf), - memo = IFNULL(:memo, memo), - is_change = IFNULL(:is_change, is_change) - WHERE tx = :tx AND output_index = :output_index", - )? - .execute(sql_args)? - == 0 - { - // It isn't there, so insert our note into the database. - db.conn - .prepare( - "UPDATE received_notes - SET account = :account, - diversifier = :diversifier, - value = :value, - rcm = :rcm, - nf = IFNULL(:nf, nf), - memo = IFNULL(:memo, memo), - is_change = IFNULL(:is_change, is_change) - WHERE tx = :tx AND output_index = :output_index", - )? - .execute(sql_args)?; - - Ok(NoteId::ReceivedNoteId(db.conn.last_insert_rowid())) - } else { - // It was there, so grab its row number. - db.conn - .prepare("SELECT id_note FROM received_notes WHERE tx = ? AND output_index = ?")? - .query_row(params![tx_ref, (output.index() as i64)], |row| { - row.get(0).map(NoteId::ReceivedNoteId) - }) - .map_err(SqliteClientError::from) - } + let mut update_ops = db.get_update_ops()?; + wallet::put_received_note(&mut update_ops, output, tx_ref) } -/// Records the incremental witness for the specified note, -/// as of the given block height. pub fn insert_witness

( db: &MutexGuard>, note_id: i64, witness: &IncrementalWitness, height: BlockHeight, ) -> Result<(), SqliteClientError> { - let mut encoded = Vec::new(); - witness.write(&mut encoded).unwrap(); - - db.conn - .prepare( - "INSERT INTO sapling_witnesses (note, block, witness) - VALUES (?, ?, ?)", - )? - .execute(params![note_id, u32::from(height), encoded])?; - - Ok(()) + let mut update_ops = db.get_update_ops()?; + wallet::insert_witness(&mut update_ops, note_id, witness, height) } -/// Removes old incremental witnesses up to the given block height. pub fn prune_witnesses

( db: &MutexGuard>, below_height: BlockHeight, ) -> Result<(), SqliteClientError> { - db.conn - .prepare("DELETE FROM sapling_witnesses WHERE block < ?")? - .execute([u32::from(below_height)])?; - Ok(()) + let mut update_ops = db.get_update_ops()?; + wallet::prune_witnesses(&mut update_ops, below_height) } -/// Marks notes that have not been mined in transactions -/// as expired, up to the given block height. pub fn update_expired_notes

( db: &MutexGuard>, height: BlockHeight, ) -> Result<(), SqliteClientError> { - db.conn - .prepare( - "UPDATE received_notes SET spent = NULL WHERE EXISTS ( - SELECT id_tx FROM transactions - WHERE id_tx = received_notes.spent AND block IS NULL AND expiry_height < ? - )", - )? - .execute([u32::from(height)])?; - Ok(()) + let mut update_ops = db.get_update_ops()?; + wallet::update_expired_notes(&mut update_ops, height) } -/// Inserts full transaction data into the database. pub fn put_tx_data

( db: &MutexGuard>, tx: &Transaction, created_at: Option, ) -> Result { - let txid = tx.txid().0.to_vec(); - - let mut raw_tx = vec![]; - tx.write(&mut raw_tx)?; - - if db - .conn - .prepare( - "UPDATE transactions - SET expiry_height = ?, raw = ? WHERE txid = ?", - )? - .execute(params![u32::from(tx.expiry_height), raw_tx, txid,])? - == 0 - { - // It isn't there, so insert our transaction into the database. - db.conn - .prepare( - "INSERT INTO transactions (txid, created, expiry_height, raw) - VALUES (?, ?, ?, ?)", - )? - .execute(params![ - txid, - created_at, - u32::from(tx.expiry_height), - raw_tx - ])?; - - Ok(db.conn.last_insert_rowid()) - } else { - // It was there, so grab its row number. - db.conn - .prepare("SELECT id_tx FROM transactions WHERE txid = ?")? - .query_row([txid], |row| row.get(0)) - .map_err(SqliteClientError::from) - } + let mut update_ops = db.get_update_ops()?; + wallet::put_tx_data(&mut update_ops, tx, created_at) } -/// Records information about a note that your wallet created. pub fn put_sent_note( db: &MutexGuard>, output: &DecryptedOutput, tx_ref: i64, ) -> Result<(), SqliteClientError> { - let output_index = output.index as i64; - let account = output.account.0 as i64; - let value = output.note.value as i64; - let to_str = encode_payment_address(db.params.hrp_sapling_payment_address(), &output.to); - - // Try updating an existing sent note. - if db - .conn - .prepare( - "UPDATE sent_notes - SET from_account = ?, address = ?, value = ?, memo = ? - WHERE tx = ? AND output_index = ?", - )? - .execute(params![ - account, - to_str, - value, - &output.memo.as_slice(), - tx_ref, - output_index - ])? - == 0 - { - // It isn't there, so insert. - insert_sent_note( - db, - tx_ref, - output.index, - output.account, - &RecipientAddress::Shielded(output.to.clone()), - Amount::from_u64(output.note.value) - .map_err(|_| SqliteClientError::CorruptedData("Note value invalid.".to_string()))?, - Some(&output.memo), - )? - } - - Ok(()) + let mut update_ops = db.get_update_ops()?; + wallet::put_sent_note(&mut update_ops, output, tx_ref) } -/// Inserts a sent note into the wallet database. -/// -/// `output_index` is the index within the transaction that contains the recipient output: -/// -/// - If `to` is a Sapling address, this is an index into the Sapling outputs of the -/// transaction. -/// - If `to` is a transparent address, this is an index into the transparent outputs of -/// the transaction. pub fn insert_sent_note( db: &MutexGuard>, tx_ref: i64, @@ -324,21 +111,14 @@ pub fn insert_sent_note( value: Amount, memo: Option<&MemoBytes>, ) -> Result<(), SqliteClientError> { - let to_str = to.encode(&db.params); - let ivalue: i64 = value.into(); - db.conn - .prepare( - "INSERT INTO sent_notes (tx, output_index, from_account, address, value, memo) - VALUES (?, ?, ?, ?, ?, ?)", - )? - .execute(params![ - tx_ref, - (output_index as i64), - account.0, - to_str, - ivalue, - memo.map(|m| m.as_slice().to_vec()), - ])?; - - Ok(()) + let mut update_ops = db.get_update_ops()?; + wallet::insert_sent_note( + &mut update_ops, + tx_ref, + output_index, + account, + to, + value, + memo, + ) } diff --git a/zcash_extras/src/lib.rs b/zcash_extras/src/lib.rs index 3470697818..b7da2d67c6 100644 --- a/zcash_extras/src/lib.rs +++ b/zcash_extras/src/lib.rs @@ -32,7 +32,7 @@ use zcash_primitives::{ #[async_trait::async_trait] pub trait WalletRead: Send + Sync + 'static { - type Error; + type Error: ToString; type NoteRef: Debug; type TxRef: Copy + Debug; From 99998ef88b3c2d082b0d7dd3717feb6b69ff8efa Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 12 Oct 2023 00:23:26 +0100 Subject: [PATCH 33/42] minor fix --- zcash_client_backend/Cargo.toml | 1 - zcash_client_backend/src/data_api.rs | 3 +-- .../src/for_async/wallet_actions.rs | 24 +++++++++---------- zcash_extras/src/lib.rs | 2 +- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index b54470fb3e..8d3c19a706 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -38,7 +38,6 @@ time = "0.3.20" [target.'cfg(target_arch = "wasm32")'.dependencies] time = { version = "0.3.20", features = ["wasm-bindgen"]} - [build-dependencies] protobuf-codegen-pure = "2.20" diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 0eef4818c8..ef4f2fd615 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -34,7 +34,7 @@ pub mod wallet; /// abstracted away from any particular data storage substrate. pub trait WalletRead { /// The type of errors produced by a wallet backend. - type Error: ToString; + type Error; /// Backend-specific note identifier. /// @@ -202,7 +202,6 @@ pub struct ReceivedTransaction<'a> { /// The purpose of this struct is to permit atomic updates of the /// wallet database when transactions are created and submitted /// to the network. -#[derive(Clone)] pub struct SentTransaction<'a> { pub tx: &'a Transaction, pub created: time::OffsetDateTime, diff --git a/zcash_client_sqlite/src/for_async/wallet_actions.rs b/zcash_client_sqlite/src/for_async/wallet_actions.rs index b1f1caab8f..bf95c35a6f 100644 --- a/zcash_client_sqlite/src/for_async/wallet_actions.rs +++ b/zcash_client_sqlite/src/for_async/wallet_actions.rs @@ -1,21 +1,19 @@ use crate::error::SqliteClientError; use crate::{wallet, NoteId, WalletDb}; -use rusqlite::params; use std::sync::MutexGuard; use zcash_client_backend::address::RecipientAddress; use zcash_client_backend::wallet::{AccountId, WalletTx}; use zcash_client_backend::DecryptedOutput; use zcash_extras::ShieldedOutput; use zcash_primitives::block::BlockHash; -use zcash_primitives::consensus; -use zcash_primitives::consensus::BlockHeight; +use zcash_primitives::consensus::{BlockHeight, Parameters}; use zcash_primitives::memo::MemoBytes; use zcash_primitives::merkle_tree::{CommitmentTree, IncrementalWitness}; use zcash_primitives::sapling::{Node, Nullifier}; use zcash_primitives::transaction::components::Amount; use zcash_primitives::transaction::Transaction; -pub fn insert_block

( +pub fn insert_block( db: &MutexGuard>, block_height: BlockHeight, block_hash: BlockHash, @@ -32,7 +30,7 @@ pub fn insert_block

( ) } -pub fn put_tx_meta( +pub fn put_tx_meta( db: &MutexGuard>, tx: &WalletTx, height: BlockHeight, @@ -41,7 +39,7 @@ pub fn put_tx_meta( wallet::put_tx_meta(&mut update_ops, tx, height) } -pub fn mark_spent

( +pub fn mark_spent( db: &MutexGuard>, tx_ref: i64, nf: &Nullifier, @@ -49,7 +47,7 @@ pub fn mark_spent

( wallet::mark_spent(&mut db.get_update_ops()?, tx_ref, nf) } -pub fn put_received_note( +pub fn put_received_note( db: &MutexGuard>, output: &T, tx_ref: i64, @@ -58,7 +56,7 @@ pub fn put_received_note( wallet::put_received_note(&mut update_ops, output, tx_ref) } -pub fn insert_witness

( +pub fn insert_witness( db: &MutexGuard>, note_id: i64, witness: &IncrementalWitness, @@ -68,7 +66,7 @@ pub fn insert_witness

( wallet::insert_witness(&mut update_ops, note_id, witness, height) } -pub fn prune_witnesses

( +pub fn prune_witnesses( db: &MutexGuard>, below_height: BlockHeight, ) -> Result<(), SqliteClientError> { @@ -76,7 +74,7 @@ pub fn prune_witnesses

( wallet::prune_witnesses(&mut update_ops, below_height) } -pub fn update_expired_notes

( +pub fn update_expired_notes( db: &MutexGuard>, height: BlockHeight, ) -> Result<(), SqliteClientError> { @@ -84,7 +82,7 @@ pub fn update_expired_notes

( wallet::update_expired_notes(&mut update_ops, height) } -pub fn put_tx_data

( +pub fn put_tx_data( db: &MutexGuard>, tx: &Transaction, created_at: Option, @@ -93,7 +91,7 @@ pub fn put_tx_data

( wallet::put_tx_data(&mut update_ops, tx, created_at) } -pub fn put_sent_note( +pub fn put_sent_note( db: &MutexGuard>, output: &DecryptedOutput, tx_ref: i64, @@ -102,7 +100,7 @@ pub fn put_sent_note( wallet::put_sent_note(&mut update_ops, output, tx_ref) } -pub fn insert_sent_note( +pub fn insert_sent_note( db: &MutexGuard>, tx_ref: i64, output_index: usize, diff --git a/zcash_extras/src/lib.rs b/zcash_extras/src/lib.rs index b7da2d67c6..3470697818 100644 --- a/zcash_extras/src/lib.rs +++ b/zcash_extras/src/lib.rs @@ -32,7 +32,7 @@ use zcash_primitives::{ #[async_trait::async_trait] pub trait WalletRead: Send + Sync + 'static { - type Error: ToString; + type Error; type NoteRef: Debug; type TxRef: Copy + Debug; From cf057b90df19488c2e6b2fa3baf08eca420bc76d Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 12 Oct 2023 00:29:56 +0100 Subject: [PATCH 34/42] change `zcash_extra` edition to 2018 --- zcash_extras/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_extras/Cargo.toml b/zcash_extras/Cargo.toml index bc4b95cb8c..109f24541a 100644 --- a/zcash_extras/Cargo.toml +++ b/zcash_extras/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_extras" version = "0.1.0" -edition = "2021" +edition = "2018" [dependencies] async-trait = "0.1.52" From 70390543e3f758388e1085f1fe2ce86eba0e8bef Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 12 Oct 2023 00:32:21 +0100 Subject: [PATCH 35/42] delete NoteId from sqlite crate --- zcash_client_sqlite/Cargo.toml | 1 - zcash_client_sqlite/src/lib.rs | 17 ----------------- 2 files changed, 18 deletions(-) diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 9f2d2874f5..a4a7b8820e 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -35,7 +35,6 @@ time = "0.3.20" [target.'cfg(target_arch = "wasm32")'.dependencies] time = { version = "0.3.20", features = ["wasm-bindgen"]} - [dev-dependencies] rand_core = "0.5.1" tempfile = "3" diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 40b7fd4fa6..157374423e 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -67,23 +67,6 @@ pub mod error; pub mod for_async; pub mod wallet; -///// A newtype wrapper for sqlite primary key values for the notes -///// table. -//#[derive(Debug, Copy, Clone)] -//pub enum NoteId { -// SentNoteId(i64), -// ReceivedNoteId(i64), -//} -// -//impl fmt::Display for NoteId { -// fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { -// match self { -// NoteId::SentNoteId(id) => write!(f, "Sent Note {}", id), -// NoteId::ReceivedNoteId(id) => write!(f, "Received Note {}", id), -// } -// } -//} - /// A wrapper for the SQLite connection to the wallet database. pub struct WalletDb

{ conn: Connection, From b3812539206e100d7fd5e70a2e42beb1cf2977cd Mon Sep 17 00:00:00 2001 From: borngraced Date: Fri, 27 Oct 2023 05:22:52 +0100 Subject: [PATCH 36/42] remove redundancy crate decl --- zcash_client_backend/Cargo.toml | 6 ++++++ zcash_client_sqlite/Cargo.toml | 7 +------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 8d3c19a706..334cedfa6e 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -47,8 +47,14 @@ rand_core = "0.5.1" rand_xorshift = "0.2" tempfile = "3.1.0" zcash_client_sqlite = { version = "0.3", path = "../zcash_client_sqlite" } + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] zcash_proofs = { version = "0.5", path = "../zcash_proofs" } +[target.'cfg(target_arch = "wasm32")'.dev-dependencies] +zcash_proofs = { version = "0.5", path = "../zcash_proofs", default-features = false, features = ["local-prover"]} + + [features] test-dependencies = ["proptest", "zcash_primitives/test-dependencies"] diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index a4a7b8820e..02b9f60efd 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -24,17 +24,12 @@ protobuf = "2.20" rand_core = "0.5.1" rusqlite = { version = "0.28", features = ["time"] } libsqlite3-sys= { version = "0.25.2", features = ["bundled"] } +time = "0.3.20" tokio = { version = "1.20", features = ["rt"] } zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } zcash_extras = { version = "0.1", path = "../zcash_extras" } zcash_primitives = { version = "0.5", path = "../zcash_primitives" } -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -time = "0.3.20" - -[target.'cfg(target_arch = "wasm32")'.dependencies] -time = { version = "0.3.20", features = ["wasm-bindgen"]} - [dev-dependencies] rand_core = "0.5.1" tempfile = "3" From 04e7b51482cc605d90b153eab749d5e0befdee62 Mon Sep 17 00:00:00 2001 From: borngraced Date: Wed, 1 Nov 2023 06:54:04 +0100 Subject: [PATCH 37/42] fix review notes --- zcash_client_backend/Cargo.toml | 1 - zcash_client_backend/src/data_api/wallet.rs | 44 +----- zcash_client_sqlite/Cargo.toml | 3 +- zcash_client_sqlite/src/for_async/mod.rs | 147 +++----------------- 4 files changed, 19 insertions(+), 176 deletions(-) diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 334cedfa6e..3616019088 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -19,7 +19,6 @@ bls12_381 = "0.3.1" bs58 = { version = "0.4", features = ["check"] } base64 = "0.13" ff = "0.8" -futures = { version = "0.3", package = "futures", features = ["compat", "async-await"] } group = "0.8" hex = "0.4" jubjub = "0.5.1" diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 82a9a9a9b5..20c2a3828f 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -2,64 +2,24 @@ use std::fmt::Debug; use zcash_primitives::{ - consensus::{self, BranchId, NetworkUpgrade}, + consensus::{self, BranchId}, memo::MemoBytes, sapling::prover::TxProver, transaction::{ builder::Builder, components::{amount::DEFAULT_FEE, Amount}, - Transaction, }, zip32::{ExtendedFullViewingKey, ExtendedSpendingKey}, }; use crate::{ address::RecipientAddress, - data_api::{error::Error, ReceivedTransaction, SentTransaction, WalletWrite}, - decrypt_transaction, + data_api::{error::Error, SentTransaction, WalletWrite}, wallet::{AccountId, OvkPolicy}, }; pub const ANCHOR_OFFSET: u32 = 10; -/// Scans a [`Transaction`] for any information that can be decrypted by the accounts in -/// the wallet, and saves it to the wallet. -pub fn decrypt_and_store_transaction( - params: &P, - data: &mut D, - tx: &Transaction, -) -> Result<(), E> -where - E: From>, - P: consensus::Parameters, - D: WalletWrite, -{ - // Fetch the ExtendedFullViewingKeys we are tracking - let extfvks = data.get_extended_full_viewing_keys()?; - - // Height is block height for mined transactions, and the "mempool height" (chain height + 1) - // for mempool transactions. - let height = data - .get_tx_height(tx.txid())? - .or(data - .block_height_extrema()? - .map(|(_, max_height)| max_height + 1)) - .or_else(|| params.activation_height(NetworkUpgrade::Sapling)) - .ok_or(Error::SaplingNotActive)?; - - let outputs = decrypt_transaction(params, height, tx, &extfvks); - if outputs.is_empty() { - Ok(()) - } else { - data.store_received_tx(&ReceivedTransaction { - tx, - outputs: &outputs, - })?; - - Ok(()) - } -} - #[allow(clippy::needless_doctest_main)] /// Creates a transaction paying the specified address from the given account. /// diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 02b9f60efd..0f3951fdfb 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -17,7 +17,6 @@ async-trait = "0.1.52" bech32 = "0.9.1" bs58 = { version = "0.4", features = ["check"] } ff = "0.8" -futures = { version = "0.3" } group = "0.8" jubjub = "0.5.1" protobuf = "2.20" @@ -25,7 +24,7 @@ rand_core = "0.5.1" rusqlite = { version = "0.28", features = ["time"] } libsqlite3-sys= { version = "0.25.2", features = ["bundled"] } time = "0.3.20" -tokio = { version = "1.20", features = ["rt"] } +tokio = { version = "1.20", features = ["rt", "rt-multi-thread"] } zcash_client_backend = { version = "0.5", path = "../zcash_client_backend" } zcash_extras = { version = "0.1", path = "../zcash_extras" } zcash_primitives = { version = "0.5", path = "../zcash_primitives" } diff --git a/zcash_client_sqlite/src/for_async/mod.rs b/zcash_client_sqlite/src/for_async/mod.rs index e3488f9fd6..810ca6c244 100644 --- a/zcash_client_sqlite/src/for_async/mod.rs +++ b/zcash_client_sqlite/src/for_async/mod.rs @@ -3,6 +3,7 @@ pub mod wallet_actions; use std::collections::HashMap; use std::path::Path; +use tokio::task::block_in_place; use zcash_client_backend::data_api::{PrunedBlock, ReceivedTransaction, SentTransaction}; use zcash_client_backend::wallet::{AccountId, SpendableNote}; use zcash_extras::{WalletRead, WalletWrite}; @@ -225,44 +226,6 @@ pub struct DataConnStmtCacheAsync

{ wallet_db: WalletDbAsync

, } -impl DataConnStmtCacheAsync

{ - fn transactionally(&mut self, f: F) -> Result - where - F: FnOnce(&mut Self) -> Result, - { - self.wallet_db - .inner - .lock() - .unwrap() - .conn - .execute("BEGIN IMMEDIATE", [])?; - match f(self) { - Ok(result) => { - self.wallet_db - .inner - .lock() - .unwrap() - .conn - .execute("COMMIT", [])?; - Ok(result) - } - Err(error) => { - match self.wallet_db.inner.lock().unwrap().conn.execute("ROLLBACK", []) { - Ok(_) => Err(error), - Err(e) => - // Panicking here is probably the right thing to do, because it - // means the database is corrupt. - panic!( - "Rollback failed with error {} while attempting to recover from error {}; database is likely corrupt.", - e, - error - ) - } - } - } - } -} - #[async_trait::async_trait] impl WalletRead for DataConnStmtCacheAsync

{ type Error = SqliteClientError; @@ -367,114 +330,36 @@ impl WalletWrite for DataConnS block: &PrunedBlock, updated_witnesses: &[(Self::NoteRef, IncrementalWitness)], ) -> Result)>, Self::Error> { - // database updates for each block are transactional - self.transactionally(|up| { - let db = up.wallet_db.inner.lock().unwrap(); - // Insert the block into the database. - wallet_actions::insert_block( - &db, - block.block_height, - block.block_hash, - block.block_time, - block.commitment_tree, - )?; - - let mut new_witnesses = vec![]; - for tx in block.transactions { - let tx_row = wallet_actions::put_tx_meta(&db, tx, block.block_height)?; - - // Mark notes as spent and remove them from the scanning cache - for spend in &tx.shielded_spends { - wallet_actions::mark_spent(&db, tx_row, &spend.nf)?; - } - - for output in &tx.shielded_outputs { - let received_note_id = wallet_actions::put_received_note(&db, output, tx_row)?; - - // Save witness for note. - new_witnesses.push((received_note_id, output.witness.clone())); - } - } - - // Insert current new_witnesses into the database. - for (received_note_id, witness) in updated_witnesses.iter().chain(new_witnesses.iter()) - { - if let NoteId::ReceivedNoteId(rnid) = *received_note_id { - wallet_actions::insert_witness(&db, rnid, witness, block.block_height)?; - } else { - return Err(SqliteClientError::InvalidNoteId); - } - } - - // Prune the stored witnesses (we only expect rollbacks of at most 100 blocks). - let below_height = if block.block_height < BlockHeight::from(100) { - BlockHeight::from(0) - } else { - block.block_height - 100 - }; - wallet_actions::prune_witnesses(&db, below_height)?; + use zcash_client_backend::data_api::WalletWrite; - // Update now-expired transactions that didn't get mined. - wallet_actions::update_expired_notes(&db, block.block_height)?; - - Ok(new_witnesses) - }) + // database updates for each block are transactional + let db = self.wallet_db.inner.lock().unwrap(); + let mut update_ops = db.get_update_ops()?; + block_in_place(|| update_ops.advance_by_block(&block, updated_witnesses)) } async fn store_received_tx( &mut self, received_tx: &ReceivedTransaction, ) -> Result { - self.transactionally(|up| { - let db = up.wallet_db.inner.lock().unwrap(); - let tx_ref = wallet_actions::put_tx_data(&db, received_tx.tx, None)?; - - for output in received_tx.outputs { - if output.outgoing { - wallet_actions::put_sent_note(&db, output, tx_ref)?; - } else { - wallet_actions::put_received_note(&db, output, tx_ref)?; - } - } + use zcash_client_backend::data_api::WalletWrite; - Ok(tx_ref) - }) + // database updates for each block are transactional + let db = self.wallet_db.inner.lock().unwrap(); + let mut update_ops = db.get_update_ops()?; + block_in_place(|| update_ops.store_received_tx(&received_tx)) } async fn store_sent_tx( &mut self, sent_tx: &SentTransaction, ) -> Result { - // Update the database atomically, to ensure the result is internally consistent. - self.transactionally(|up| { - let db = up.wallet_db.inner.lock().unwrap(); - let tx_ref = wallet_actions::put_tx_data(&db, sent_tx.tx, Some(sent_tx.created))?; - - // Mark notes as spent. - // - // This locks the notes so they aren't selected again by a subsequent call to - // create_spend_to_address() before this transaction has been mined (at which point the notes - // get re-marked as spent). - // - // Assumes that create_spend_to_address() will never be called in parallel, which is a - // reasonable assumption for a light client such as a mobile phone. - for spend in &sent_tx.tx.shielded_spends { - wallet_actions::mark_spent(&db, tx_ref, &spend.nullifier)?; - } + use zcash_client_backend::data_api::WalletWrite; - wallet_actions::insert_sent_note( - &db, - tx_ref, - sent_tx.output_index, - sent_tx.account, - sent_tx.recipient_address, - sent_tx.value, - sent_tx.memo.as_ref(), - )?; - - // Return the row number of the transaction, so the caller can fetch it for sending. - Ok(tx_ref) - }) + // Update the database atomically, to ensure the result is internally consistent. + let db = self.wallet_db.inner.lock().unwrap(); + let mut update_ops = db.get_update_ops()?; + block_in_place(|| update_ops.store_sent_tx(&sent_tx)) } async fn rewind_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error> { From 1b20142caf8fbc34630dcee10bf068f31b9811bd Mon Sep 17 00:00:00 2001 From: borngraced Date: Wed, 1 Nov 2023 15:08:49 +0100 Subject: [PATCH 38/42] fix review notes --- zcash_client_backend/Cargo.toml | 1 - zcash_client_backend/src/data_api.rs | 1 - zcash_client_backend/src/data_api/wallet.rs | 42 +++++++++++++++++++++ zcash_client_sqlite/src/for_async/mod.rs | 32 ++++++++-------- zcash_client_sqlite/src/lib.rs | 1 - zcash_extras/Cargo.toml | 4 ++ zcash_extras/src/lib.rs | 20 ++-------- 7 files changed, 67 insertions(+), 34 deletions(-) diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 3616019088..7660f01f04 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -53,7 +53,6 @@ zcash_proofs = { version = "0.5", path = "../zcash_proofs" } [target.'cfg(target_arch = "wasm32")'.dev-dependencies] zcash_proofs = { version = "0.5", path = "../zcash_proofs", default-features = false, features = ["local-prover"]} - [features] test-dependencies = ["proptest", "zcash_primitives/test-dependencies"] diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index ef4f2fd615..166fd80cba 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -178,7 +178,6 @@ pub trait WalletRead { /// The subset of information that is relevant to this wallet that has been /// decrypted and extracted from a [CompactBlock]. -#[derive(Clone)] pub struct PrunedBlock<'a> { pub block_height: BlockHeight, pub block_hash: BlockHash, diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 20c2a3828f..090bb14a79 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1,6 +1,8 @@ //! Functions for scanning the chain and extracting relevant information. use std::fmt::Debug; +use zcash_primitives::consensus::NetworkUpgrade; +use zcash_primitives::transaction::Transaction; use zcash_primitives::{ consensus::{self, BranchId}, memo::MemoBytes, @@ -12,14 +14,54 @@ use zcash_primitives::{ zip32::{ExtendedFullViewingKey, ExtendedSpendingKey}, }; +use crate::data_api::ReceivedTransaction; use crate::{ address::RecipientAddress, data_api::{error::Error, SentTransaction, WalletWrite}, + decrypt_transaction, wallet::{AccountId, OvkPolicy}, }; pub const ANCHOR_OFFSET: u32 = 10; +/// Scans a [`Transaction`] for any information that can be decrypted by the accounts in +/// the wallet, and saves it to the wallet. +pub fn decrypt_and_store_transaction( + params: &P, + data: &mut D, + tx: &Transaction, +) -> Result<(), E> +where + E: From>, + P: consensus::Parameters, + D: WalletWrite, +{ + // Fetch the ExtendedFullViewingKeys we are tracking + let extfvks = data.get_extended_full_viewing_keys()?; + + // Height is block height for mined transactions, and the "mempool height" (chain height + 1) + // for mempool transactions. + let height = data + .get_tx_height(tx.txid())? + .or(data + .block_height_extrema()? + .map(|(_, max_height)| max_height + 1)) + .or_else(|| params.activation_height(NetworkUpgrade::Sapling)) + .ok_or(Error::SaplingNotActive)?; + + let outputs = decrypt_transaction(params, height, tx, &extfvks); + if outputs.is_empty() { + Ok(()) + } else { + data.store_received_tx(&ReceivedTransaction { + tx, + outputs: &outputs, + })?; + + Ok(()) + } +} + #[allow(clippy::needless_doctest_main)] /// Creates a transaction paying the specified address from the given account. /// diff --git a/zcash_client_sqlite/src/for_async/mod.rs b/zcash_client_sqlite/src/for_async/mod.rs index 810ca6c244..ebe60393a0 100644 --- a/zcash_client_sqlite/src/for_async/mod.rs +++ b/zcash_client_sqlite/src/for_async/mod.rs @@ -4,6 +4,7 @@ pub mod wallet_actions; use std::collections::HashMap; use std::path::Path; use tokio::task::block_in_place; +use zcash_client_backend::data_api::WalletWrite as WalletWriteSync; use zcash_client_backend::data_api::{PrunedBlock, ReceivedTransaction, SentTransaction}; use zcash_client_backend::wallet::{AccountId, SpendableNote}; use zcash_extras::{WalletRead, WalletWrite}; @@ -330,36 +331,37 @@ impl WalletWrite for DataConnS block: &PrunedBlock, updated_witnesses: &[(Self::NoteRef, IncrementalWitness)], ) -> Result)>, Self::Error> { - use zcash_client_backend::data_api::WalletWrite; - // database updates for each block are transactional - let db = self.wallet_db.inner.lock().unwrap(); - let mut update_ops = db.get_update_ops()?; - block_in_place(|| update_ops.advance_by_block(&block, updated_witnesses)) + block_in_place(|| { + let db = self.wallet_db.inner.lock().unwrap(); + let mut update_ops = db.get_update_ops()?; + update_ops.advance_by_block(&block, updated_witnesses) + }) } async fn store_received_tx( &mut self, received_tx: &ReceivedTransaction, ) -> Result { - use zcash_client_backend::data_api::WalletWrite; - // database updates for each block are transactional - let db = self.wallet_db.inner.lock().unwrap(); - let mut update_ops = db.get_update_ops()?; - block_in_place(|| update_ops.store_received_tx(&received_tx)) + block_in_place(|| { + let db = self.wallet_db.inner.lock().unwrap(); + let mut update_ops = db.get_update_ops()?; + update_ops.store_received_tx(&received_tx) + }) } async fn store_sent_tx( &mut self, sent_tx: &SentTransaction, ) -> Result { - use zcash_client_backend::data_api::WalletWrite; - // Update the database atomically, to ensure the result is internally consistent. - let db = self.wallet_db.inner.lock().unwrap(); - let mut update_ops = db.get_update_ops()?; - block_in_place(|| update_ops.store_sent_tx(&sent_tx)) + + block_in_place(|| { + let db = self.wallet_db.inner.lock().unwrap(); + let mut update_ops = db.get_update_ops()?; + update_ops.store_sent_tx(&sent_tx) + }) } async fn rewind_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error> { diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 157374423e..91561417eb 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -35,7 +35,6 @@ extern crate core; use std::collections::HashMap; -//use std::fmt; use std::path::Path; use rusqlite::{Connection, Statement}; diff --git a/zcash_extras/Cargo.toml b/zcash_extras/Cargo.toml index 109f24541a..aa07c7ca88 100644 --- a/zcash_extras/Cargo.toml +++ b/zcash_extras/Cargo.toml @@ -3,6 +3,9 @@ name = "zcash_extras" version = "0.1.0" edition = "2018" +[lib] +doctest = false + [dependencies] async-trait = "0.1.52" group = "0.8" @@ -18,3 +21,4 @@ time = "0.3.20" [target.'cfg(target_arch = "wasm32")'.dependencies] time = { version = "0.3.20", features = ["wasm-bindgen"]} + diff --git a/zcash_extras/src/lib.rs b/zcash_extras/src/lib.rs index 3470697818..73f357b619 100644 --- a/zcash_extras/src/lib.rs +++ b/zcash_extras/src/lib.rs @@ -280,23 +280,11 @@ impl fmt::Display for NoteId { } } } -#[cfg(feature = "mainnet")] -pub(crate) fn network() -> Network { - Network::MainNetwork -} -#[cfg(not(feature = "mainnet"))] -pub(crate) fn network() -> Network { +pub(crate) fn test_network() -> Network { Network::TestNetwork } -#[cfg(feature = "mainnet")] -pub(crate) fn sapling_activation_height() -> BlockHeight { - Network::MainNetwork - .activation_height(NetworkUpgrade::Sapling) - .unwrap() -} - /// Create a fake CompactBlock at the given height, containing a single output paying /// the given address. Returns the CompactBlock and the nullifier for the new note. pub fn fake_compact_block( @@ -309,7 +297,7 @@ pub fn fake_compact_block( // Create a fake Note for the account let mut rng = OsRng; - let rseed = generate_random_rseed(&network(), height, &mut rng); + let rseed = generate_random_rseed(&test_network(), height, &mut rng); let note = Note { g_d: to.diversifier().g_d().unwrap(), pk_d: *to.pk_d(), @@ -357,7 +345,7 @@ pub fn fake_compact_block_spending( value: Amount, ) -> CompactBlock { let mut rng = OsRng; - let rseed = generate_random_rseed(&network(), height, &mut rng); + let rseed = generate_random_rseed(&test_network(), height, &mut rng); // Create a fake CompactBlock containing the note let mut cspend = CompactSpend::new(); @@ -397,7 +385,7 @@ pub fn fake_compact_block_spending( // Create a fake Note for the change ctx.outputs.push({ let change_addr = extfvk.default_address().unwrap().1; - let rseed = generate_random_rseed(&network(), height, &mut rng); + let rseed = generate_random_rseed(&test_network(), height, &mut rng); let note = Note { g_d: change_addr.diversifier().g_d().unwrap(), pk_d: *change_addr.pk_d(), From 97a64eaf22b6fa1517238f93e06f92f69f36733e Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 2 Nov 2023 04:13:20 +0100 Subject: [PATCH 39/42] make LocalTxProver fields private --- zcash_proofs/src/prover.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zcash_proofs/src/prover.rs b/zcash_proofs/src/prover.rs index caf81ddc73..a6a588cf86 100644 --- a/zcash_proofs/src/prover.rs +++ b/zcash_proofs/src/prover.rs @@ -21,9 +21,9 @@ use crate::{default_params_folder, SAPLING_OUTPUT_NAME, SAPLING_SPEND_NAME}; /// An implementation of [`TxProver`] using Sapling Spend and Output parameters from /// locally-accessible paths. pub struct LocalTxProver { - pub spend_params: Parameters, - pub spend_vk: PreparedVerifyingKey, - pub output_params: Parameters, + spend_params: Parameters, + spend_vk: PreparedVerifyingKey, + output_params: Parameters, } impl LocalTxProver { From cfbe0a0c26f2a84bf2304f24b94eb366f02ae086 Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 2 Nov 2023 05:07:24 +0100 Subject: [PATCH 40/42] fix sql init module review notes --- zcash_client_sqlite/src/for_async/init.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/zcash_client_sqlite/src/for_async/init.rs b/zcash_client_sqlite/src/for_async/init.rs index 91d9a0339d..3bdaf7ffcb 100644 --- a/zcash_client_sqlite/src/for_async/init.rs +++ b/zcash_client_sqlite/src/for_async/init.rs @@ -12,7 +12,7 @@ pub async fn init_wallet_db( where P: Clone + Send + Sync, { - let wdb = wdb.inner.clone(); + let wdb = wdb.inner(); async_blocking(move || { let wdb = wdb.lock().unwrap(); wallet::init::init_wallet_db(&wdb) @@ -27,9 +27,8 @@ pub async fn init_accounts_table( where P: Clone + Send + Sync, { - let wdb = wdb.inner.clone(); + let wdb = wdb.inner(); let extfvks = extfvks.to_vec(); - async_blocking(move || { let wdb = wdb.lock().unwrap(); wallet::init::init_accounts_table(&wdb, &extfvks) @@ -47,9 +46,8 @@ pub async fn init_blocks_table( where P: Clone + Send + Sync, { - let wdb = wdb.inner.clone(); - let sapling_tree = sapling_tree.to_vec().clone(); - + let wdb = wdb.inner(); + let sapling_tree = sapling_tree.to_vec(); async_blocking(move || { let wdb = wdb.lock().unwrap(); wallet::init::init_blocks_table(&wdb, height, hash, time, &sapling_tree) From faed2b8773280c044d8e2497e5f66c26b53461d4 Mon Sep 17 00:00:00 2001 From: borngraced Date: Thu, 2 Nov 2023 05:09:23 +0100 Subject: [PATCH 41/42] remove decrypt_and_store_transaction --- zcash_extras/src/wallet.rs | 43 -------------------------------------- 1 file changed, 43 deletions(-) diff --git a/zcash_extras/src/wallet.rs b/zcash_extras/src/wallet.rs index 0b14fb4cb0..63fbd72a7f 100644 --- a/zcash_extras/src/wallet.rs +++ b/zcash_extras/src/wallet.rs @@ -21,49 +21,6 @@ use zcash_client_backend::{ wallet::{AccountId, OvkPolicy}, }; -pub const ANCHOR_OFFSET: u32 = 10; - -/// Scans a [`Transaction`] for any information that can be decrypted by the accounts in -/// the wallet, and saves it to the wallet. -pub async fn decrypt_and_store_transaction( - params: &P, - data: &mut D, - tx: &Transaction, -) -> Result<(), E> -where - E: From>, - P: consensus::Parameters, - D: WalletWrite, -{ - // Fetch the ExtendedFullViewingKeys we are tracking - let extfvks = data.get_extended_full_viewing_keys().await?; - - // Height is block height for mined transactions, and the "mempool height" (chain height + 1) - // for mempool transactions. - let height = data - .get_tx_height(tx.txid()) - .await? - .or(data - .block_height_extrema() - .await? - .map(|(_, max_height)| max_height + 1)) - .or_else(|| params.activation_height(NetworkUpgrade::Sapling)) - .ok_or(Error::SaplingNotActive)?; - - let outputs = decrypt_transaction(params, height, tx, &extfvks); - if outputs.is_empty() { - Ok(()) - } else { - data.store_received_tx(&ReceivedTransaction { - tx, - outputs: &outputs, - }) - .await?; - - Ok(()) - } -} - #[allow(clippy::needless_doctest_main)] /// Creates a transaction paying the specified address from the given account. /// From 8c97f561d233a2c5b5e3da4b6f29bd05be052166 Mon Sep 17 00:00:00 2001 From: borngraced Date: Tue, 7 Nov 2023 12:28:04 +0100 Subject: [PATCH 42/42] fix review notes --- zcash_client_sqlite/src/chain.rs | 6 +- zcash_client_sqlite/src/lib.rs | 155 +-------------------- zcash_client_sqlite/src/wallet/transact.rs | 7 +- zcash_extras/src/wallet.rs | 6 +- 4 files changed, 11 insertions(+), 163 deletions(-) diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index b416ebe889..f3f88c33a1 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -79,14 +79,12 @@ mod tests { chain::{scan_cached_blocks, validate_chain}, error::{ChainInvalid, Error}, }; + use zcash_extras::{fake_compact_block, fake_compact_block_spending}; use crate::{ chain::init::init_cache_database, error::SqliteClientError, - tests::{ - self, fake_compact_block, fake_compact_block_spending, insert_into_cache, - sapling_activation_height, - }, + tests::{self, insert_into_cache, sapling_activation_height}, wallet::{ get_balance, init::{init_accounts_table, init_wallet_db}, diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 91561417eb..97064a64cc 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -544,27 +544,12 @@ fn address_from_extfvk( #[cfg(test)] mod tests { - use ff::PrimeField; - use group::GroupEncoding; use protobuf::Message; - use rand_core::{OsRng, RngCore}; use rusqlite::params; - use zcash_client_backend::proto::compact_formats::{ - CompactBlock, CompactOutput, CompactSpend, CompactTx, - }; - - use zcash_primitives::{ - block::BlockHash, - consensus::{BlockHeight, Network, NetworkUpgrade, Parameters}, - memo::MemoBytes, - sapling::{ - note_encryption::sapling_note_encryption, util::generate_random_rseed, Note, Nullifier, - PaymentAddress, - }, - transaction::components::Amount, - zip32::ExtendedFullViewingKey, - }; + use zcash_client_backend::proto::compact_formats::CompactBlock; + + use zcash_primitives::consensus::{BlockHeight, Network, NetworkUpgrade, Parameters}; use super::BlockDb; @@ -592,140 +577,6 @@ mod tests { .unwrap() } - /// Create a fake CompactBlock at the given height, containing a single output paying - /// the given address. Returns the CompactBlock and the nullifier for the new note. - pub fn fake_compact_block( - height: BlockHeight, - prev_hash: BlockHash, - extfvk: ExtendedFullViewingKey, - value: Amount, - ) -> (CompactBlock, Nullifier) { - let to = extfvk.default_address().unwrap().1; - - // Create a fake Note for the account - let mut rng = OsRng; - let rseed = generate_random_rseed(&network(), height, &mut rng); - let note = Note { - g_d: to.diversifier().g_d().unwrap(), - pk_d: *to.pk_d(), - value: value.into(), - rseed, - }; - let encryptor = sapling_note_encryption::<_, Network>( - Some(extfvk.fvk.ovk), - note.clone(), - to, - MemoBytes::empty(), - &mut rng, - ); - let cmu = note.cmu().to_repr().as_ref().to_vec(); - let epk = encryptor.epk().to_bytes().to_vec(); - let enc_ciphertext = encryptor.encrypt_note_plaintext(); - - // Create a fake CompactBlock containing the note - let mut cout = CompactOutput::new(); - cout.set_cmu(cmu); - cout.set_epk(epk); - cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); - let mut ctx = CompactTx::new(); - let mut txid = vec![0; 32]; - rng.fill_bytes(&mut txid); - ctx.set_hash(txid); - ctx.outputs.push(cout); - let mut cb = CompactBlock::new(); - cb.set_height(u64::from(height)); - cb.hash.resize(32, 0); - rng.fill_bytes(&mut cb.hash); - cb.prevHash.extend_from_slice(&prev_hash.0); - cb.vtx.push(ctx); - (cb, note.nf(&extfvk.fvk.vk, 0)) - } - - /// Create a fake CompactBlock at the given height, spending a single note from the - /// given address. - pub fn fake_compact_block_spending( - height: BlockHeight, - prev_hash: BlockHash, - (nf, in_value): (Nullifier, Amount), - extfvk: ExtendedFullViewingKey, - to: PaymentAddress, - value: Amount, - ) -> CompactBlock { - let mut rng = OsRng; - let rseed = generate_random_rseed(&network(), height, &mut rng); - - // Create a fake CompactBlock containing the note - let mut cspend = CompactSpend::new(); - cspend.set_nf(nf.to_vec()); - let mut ctx = CompactTx::new(); - let mut txid = vec![0; 32]; - rng.fill_bytes(&mut txid); - ctx.set_hash(txid); - ctx.spends.push(cspend); - - // Create a fake Note for the payment - ctx.outputs.push({ - let note = Note { - g_d: to.diversifier().g_d().unwrap(), - pk_d: *to.pk_d(), - value: value.into(), - rseed, - }; - let encryptor = sapling_note_encryption::<_, Network>( - Some(extfvk.fvk.ovk), - note.clone(), - to, - MemoBytes::empty(), - &mut rng, - ); - let cmu = note.cmu().to_repr().as_ref().to_vec(); - let epk = encryptor.epk().to_bytes().to_vec(); - let enc_ciphertext = encryptor.encrypt_note_plaintext(); - - let mut cout = CompactOutput::new(); - cout.set_cmu(cmu); - cout.set_epk(epk); - cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); - cout - }); - - // Create a fake Note for the change - ctx.outputs.push({ - let change_addr = extfvk.default_address().unwrap().1; - let rseed = generate_random_rseed(&network(), height, &mut rng); - let note = Note { - g_d: change_addr.diversifier().g_d().unwrap(), - pk_d: *change_addr.pk_d(), - value: (in_value - value).into(), - rseed, - }; - let encryptor = sapling_note_encryption::<_, Network>( - Some(extfvk.fvk.ovk), - note.clone(), - change_addr, - MemoBytes::empty(), - &mut rng, - ); - let cmu = note.cmu().to_repr().as_ref().to_vec(); - let epk = encryptor.epk().to_bytes().to_vec(); - let enc_ciphertext = encryptor.encrypt_note_plaintext(); - - let mut cout = CompactOutput::new(); - cout.set_cmu(cmu); - cout.set_epk(epk); - cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); - cout - }); - - let mut cb = CompactBlock::new(); - cb.set_height(u64::from(height)); - cb.hash.resize(32, 0); - rng.fill_bytes(&mut cb.hash); - cb.prevHash.extend_from_slice(&prev_hash.0); - cb.vtx.push(ctx); - cb - } - /// Insert a fake CompactBlock into the cache DB. pub(crate) fn insert_into_cache(db_cache: &BlockDb, cb: &CompactBlock) { let cb_bytes = cb.write_to_bytes().unwrap(); diff --git a/zcash_client_sqlite/src/wallet/transact.rs b/zcash_client_sqlite/src/wallet/transact.rs index e76f5eec97..32a7cf3946 100644 --- a/zcash_client_sqlite/src/wallet/transact.rs +++ b/zcash_client_sqlite/src/wallet/transact.rs @@ -166,10 +166,11 @@ mod tests { data_api::{chain::scan_cached_blocks, wallet::create_spend_to_address, WalletRead}, wallet::OvkPolicy, }; + use zcash_extras::fake_compact_block; use crate::{ chain::init::init_cache_database, - tests::{self, fake_compact_block, insert_into_cache, sapling_activation_height}, + tests::{self, insert_into_cache, sapling_activation_height}, wallet::{ get_balance, get_balance_at, init::{init_accounts_table, init_blocks_table, init_wallet_db}, @@ -610,7 +611,7 @@ mod tests { .query_row( "SELECT raw FROM transactions WHERE id_tx = ?", - [tx_row], + &[tx_row], |row| row.get(0), ) .unwrap(); @@ -623,7 +624,7 @@ mod tests { .query_row( "SELECT output_index FROM sent_notes WHERE tx = ?", - [tx_row], + &[tx_row], |row| row.get(0), ) .unwrap(); diff --git a/zcash_extras/src/wallet.rs b/zcash_extras/src/wallet.rs index 63fbd72a7f..c097bd99ba 100644 --- a/zcash_extras/src/wallet.rs +++ b/zcash_extras/src/wallet.rs @@ -2,13 +2,12 @@ use std::fmt::{Debug, Display}; use zcash_primitives::{ - consensus::{self, BranchId, NetworkUpgrade}, + consensus::{self, BranchId}, memo::MemoBytes, sapling::prover::TxProver, transaction::{ builder::Builder, components::{amount::DEFAULT_FEE, Amount}, - Transaction, }, zip32::{ExtendedFullViewingKey, ExtendedSpendingKey}, }; @@ -16,8 +15,7 @@ use zcash_primitives::{ use crate::WalletWrite; use zcash_client_backend::{ address::RecipientAddress, - data_api::{error::Error, ReceivedTransaction, SentTransaction}, - decrypt_transaction, + data_api::{error::Error, SentTransaction}, wallet::{AccountId, OvkPolicy}, };