feat: SpendingKey self derivation, parallel iteration

Primary features:
1. SpendingKey can now derive its own children without any wallet state.
   This enables RPC clients to derive keys themselves.

2. Derivation iterators.  SpendingKeyIter and SpendingKeyParallelIter.
   The latter leverages rayon to utilize all cpu cores.

3. /known_keys rpc now returns a SpendingKeyRange instead of a list of
   addresses.  This is a big perf win for large wallets.

changelog:

  refactor spending-key types so each key can derive its own children
  known_keys rpcs now return iterators instead of Vec
  DerivationIndex: u64 -> u128.
  fix premine_distribution(). separate test vs non-test.  make it aware of current network
  impl IntoIterator for SpendingKey
  GenerationReceivingAddress::derive_from_seed() -> from_seed()
  impl rayon parallel iterators
  SpendingKeyIter now starts at child 0, instead of parent_key.  fix some DoubleEndedIter issues
  iteration tests
  add test double_ended_range_iterator_meet_middle
  add test double_ended_iterator_to_first_elem
  add tests double_ended_range_iterator_to_first_elem, range_iterator_to_last_elem, iterator_nth
  separate SpendingKeyParallelIter from SpendingKeyIter
  add par_iter tests
  iterator accepts any impl RangeBounds
  tests to validate RangeBounds conversions
  known_keys() rpc returns SpendingKeyRange instead of SpendingKeyIter
  only perform extra GenerationSpendingKey assertions for debug build. add comments

src/bin/dashboard_src/address_screen.rs

@@ -10,7 +10,7 @@ use crossterm::event::KeyEventKind;
 use itertools::Itertools;
 use neptune_cash::config_models::network::Network;
 use neptune_cash::models::state::wallet::address::KeyType;
-use neptune_cash::models::state::wallet::address::SpendingKey;
+use neptune_cash::models::state::wallet::address::SpendingKeyRange;
 use neptune_cash::rpc_server::RPCClient;
 use ratatui::layout::Constraint;
 use ratatui::layout::Margin;
@@ -34,7 +34,7 @@ use unicode_width::UnicodeWidthStr;
 use super::dashboard_app::DashboardEvent;
 use super::screen::Screen;
 
-type AddressUpdate = SpendingKey;
+type AddressUpdate = SpendingKeyRange;
 type AddressUpdateArc = Arc<std::sync::Mutex<Vec<AddressUpdate>>>;
 type DashboardEventArc = Arc<std::sync::Mutex<Option<DashboardEvent>>>;
 type JoinHandleArc = Arc<Mutex<JoinHandle<()>>>;
@@ -260,19 +260,25 @@ impl Widget for AddressScreen {
         let selected_style = style.add_modifier(Modifier::REVERSED);
         let header = vec!["type", "address (abbreviated)"];
 
+        // derive all the known keys and generate data matrix.
+        //
+        // todo: only derive and render the keys that will actually be displayed.
+        // eg if we have 5000 known keys and 10 are displayed at a time, we should
+        // only derive those 10 keys.
         let matrix = self
             .data
             .lock()
             .unwrap()
             .iter()
-            .rev()
-            .map(|key| {
-                vec![
-                    KeyType::from(key).to_string(),
-                    key.to_address()
-                        .to_display_bech32m_abbreviated(self.network)
-                        .unwrap(),
-                ]
+            .flat_map(|known_keys| {
+                known_keys.iter().map(|key| {
+                    vec![
+                        KeyType::from(&key).to_string(),
+                        key.to_address()
+                            .to_display_bech32m_abbreviated(self.network)
+                            .unwrap(),
+                    ]
+                })
             })
             .collect_vec();
         let ncols = header.len();

src/models/blockchain/block/block_height.rs

@@ -131,6 +131,7 @@ mod test {
     use tracing_test::traced_test;
 
     use super::*;
+    use crate::config_models::network::Network;
     use crate::models::blockchain::block::Block;
     use crate::models::blockchain::block::TARGET_BLOCK_INTERVAL;
     use crate::models::blockchain::type_scripts::neptune_coins::NeptuneCoins;
@@ -159,6 +160,7 @@ mod test {
 
     #[test]
     fn asymptotic_limit_is_42_million() {
+        let network = Network::Alpha;
         let generation_0_subsidy = Block::block_subsidy(BlockHeight::genesis().next());
 
         // Genesis block does not contain block subsidy so it must be subtracted
@@ -182,7 +184,7 @@ mod test {
         assert!(relative_premine < 0.0198, "Premine may not exceed promise");
 
         // Designated premine is less than or equal to allocation
-        let actual_premine = Block::premine_distribution()
+        let actual_premine = Block::premine_distribution(network)
             .iter()
             .map(|(_receiving_address, amount)| *amount)
             .sum::<NeptuneCoins>();

src/models/blockchain/transaction/primitive_witness.rs

@@ -355,15 +355,15 @@ impl PrimitiveWitness {
                     amount.div_two();
                 }
                 let liquid_utxo = Utxo::new(
-                    generation_address::GenerationSpendingKey::derive_from_seed(*seed)
+                    generation_address::GenerationSpendingKey::from_seed(*seed)
                         .to_address()
                         .lock_script(),
                     amount.to_native_coins(),
                 );
                 let mut utxos = vec![liquid_utxo];
                 if let Some(release_date) = timelock_until {
                     let timelocked_utxo = Utxo::new(
-                        generation_address::GenerationSpendingKey::derive_from_seed(*seed)
+                        generation_address::GenerationSpendingKey::from_seed(*seed)
                             .to_address()
                             .lock_script(),
                         [
@@ -906,7 +906,7 @@ pub mod neptune_arbitrary {
             let input_spending_keys = address_seeds
                 .iter()
                 .map(|address_seed| {
-                    generation_address::GenerationSpendingKey::derive_from_seed(*address_seed)
+                    generation_address::GenerationSpendingKey::from_seed(*address_seed)
                 })
                 .collect_vec();
 

src/models/state/mod.rs

@@ -3145,7 +3145,7 @@ mod global_state_tests {
             };
 
             // in alice wallet: send pre-mined funds to bob
-            let an_address = GenerationReceivingAddress::derive_from_seed(rng.gen());
+            let an_address = GenerationReceivingAddress::from_seed(rng.gen());
             let block_1 = {
                 let vm_job_queue = alice_state_lock.vm_job_queue().clone();
                 let mut alice_state_mut = alice_state_lock.lock_guard_mut().await;

src/models/state/wallet/address.rs

@@ -1,5 +1,7 @@
 mod address_type;
 mod common;
+mod key_iter;
+mod spending_key_range;
 
 pub mod encrypted_utxo_notification;
 pub mod generation_address;
@@ -13,3 +15,8 @@ pub use address_type::ReceivingAddress;
 /// SpendingKey abstracts over any spending key type and should be used
 /// wherever possible.
 pub use address_type::SpendingKey;
+/// Index for deriving child-keys, of any key-type.
+pub use common::DerivationIndex;
+pub use key_iter::*;
+/// spending keys that have been used (derived) by the wallet to date.
+pub use spending_key_range::SpendingKeyRange;

src/models/state/wallet/address/address_type.rs

@@ -3,14 +3,17 @@
 use anyhow::bail;
 use anyhow::Result;
 use arbitrary::Arbitrary;
+use rayon::prelude::IntoParallelIterator;
 use serde::Deserialize;
 use serde::Serialize;
 use tasm_lib::triton_vm::prelude::Digest;
 use tracing::warn;
 
 use super::common;
 use super::generation_address;
+use super::par_iter::SpendingKeyParallelIter;
 use super::symmetric_key;
+use super::SpendingKeyIter;
 use crate::config_models::network::Network;
 use crate::models::blockchain::transaction::lock_script::LockScript;
 use crate::models::blockchain::transaction::lock_script::LockScriptAndWitness;
@@ -29,7 +32,7 @@ use crate::BFieldElement;
 // actually stored in PublicAnnouncement.
 
 /// enumerates available cryptographic key implementations for sending and receiving funds.
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq)]
 #[repr(u8)]
 pub enum KeyType {
     /// [generation_address] built on [crate::prelude::twenty_first::math::lattice::kem]
@@ -363,6 +366,24 @@ impl std::hash::Hash for SpendingKey {
     }
 }
 
+impl IntoIterator for SpendingKey {
+    type Item = Self;
+    type IntoIter = SpendingKeyIter;
+
+    fn into_iter(self) -> Self::IntoIter {
+        SpendingKeyIter::from(self)
+    }
+}
+
+impl IntoParallelIterator for SpendingKey {
+    type Iter = SpendingKeyParallelIter;
+    type Item = Self;
+
+    fn into_par_iter(self) -> Self::Iter {
+        SpendingKeyParallelIter::from(self.into_iter())
+    }
+}
+
 impl From<generation_address::GenerationSpendingKey> for SpendingKey {
     fn from(key: generation_address::GenerationSpendingKey) -> Self {
         Self::Generation(key)
@@ -376,6 +397,30 @@ impl From<symmetric_key::SymmetricKey> for SpendingKey {
 }
 
 impl SpendingKey {
+    /// generates a new SpendingKey of requested `key_type` from `seed`
+    ///
+    /// perf:
+    ///   cheap for KeyType::Symmetric (only copies seed)
+    ///   not cheap for KeyType::Generation (performs several hash ops)
+    pub fn from_seed(seed: Digest, key_type: KeyType) -> Self {
+        match key_type {
+            KeyType::Generation => {
+                generation_address::GenerationSpendingKey::from_seed(seed).into()
+            }
+            KeyType::Symmetric => symmetric_key::SymmetricKey::from_seed(seed).into(),
+        }
+    }
+
+    /// derives a child key from this key
+    ///
+    /// The first child is at index 0, and so on.
+    pub fn derive_child(&self, index: common::DerivationIndex) -> SpendingKey {
+        match self {
+            Self::Generation(k) => k.derive_child(index).into(),
+            Self::Symmetric(k) => k.derive_child(index).into(),
+        }
+    }
+
     /// returns the address that corresponds to this spending key.
     pub fn to_address(&self) -> ReceivingAddress {
         match self {
@@ -468,6 +513,20 @@ impl SpendingKey {
             })
     }
 
+    pub fn into_range_iter(
+        self,
+        range: impl std::ops::RangeBounds<common::DerivationIndex>,
+    ) -> SpendingKeyIter {
+        SpendingKeyIter::new_range(self, range)
+    }
+
+    pub fn into_par_range_iter(
+        self,
+        range: impl std::ops::RangeBounds<common::DerivationIndex>,
+    ) -> SpendingKeyParallelIter {
+        SpendingKeyParallelIter::from(SpendingKeyIter::new_range(self, range))
+    }
+
     /// converts a result into an Option and logs a warning on any error
     fn ok_warn<T>(&self, result: Result<T>) -> Option<T> {
         match result {
@@ -510,7 +569,7 @@ mod test {
     /// tests scanning for announced utxos with an asymmetric (generation) key
     #[proptest]
     fn scan_for_announced_utxos_generation(#[strategy(arb())] seed: Digest) {
-        worker::scan_for_announced_utxos(GenerationSpendingKey::derive_from_seed(seed).into())
+        worker::scan_for_announced_utxos(SpendingKey::from_seed(seed, KeyType::Generation))
     }
 
     /// tests encrypting and decrypting with a symmetric key
@@ -522,7 +581,7 @@ mod test {
     /// tests encrypting and decrypting with an asymmetric (generation) key
     #[proptest]
     fn test_encrypt_decrypt_generation(#[strategy(arb())] seed: Digest) {
-        worker::test_encrypt_decrypt(GenerationSpendingKey::derive_from_seed(seed).into())
+        worker::test_encrypt_decrypt(GenerationSpendingKey::from_seed(seed).into())
     }
 
     /// tests keygen, sign, and verify with a symmetric key
@@ -538,8 +597,8 @@ mod test {
     #[proptest]
     fn test_keygen_sign_verify_generation(#[strategy(arb())] seed: Digest) {
         worker::test_keypair_validity(
-            GenerationSpendingKey::derive_from_seed(seed).into(),
-            GenerationReceivingAddress::derive_from_seed(seed).into(),
+            SpendingKey::from_seed(seed, KeyType::Generation),
+            SpendingKey::from_seed(seed, KeyType::Generation).to_address(),
         );
     }
 
@@ -552,7 +611,7 @@ mod test {
     /// tests bech32m serialize, deserialize with an asymmetric (generation) key
     #[proptest]
     fn test_bech32m_conversion_generation(#[strategy(arb())] seed: Digest) {
-        worker::test_bech32m_conversion(GenerationReceivingAddress::derive_from_seed(seed).into());
+        worker::test_bech32m_conversion(GenerationReceivingAddress::from_seed(seed).into());
     }
 
     mod worker {

src/models/state/wallet/address/common.rs

@@ -13,6 +13,8 @@ use crate::models::blockchain::transaction::PublicAnnouncement;
 use crate::models::state::wallet::utxo_notification::UtxoNotificationPayload;
 use crate::prelude::twenty_first;
 
+pub type DerivationIndex = u128;
+
 /// returns human-readable-prefix for the given network
 pub(crate) fn network_hrp_char(network: Network) -> char {
     match network {

src/models/state/wallet/address/generation_address.rs

@@ -25,9 +25,11 @@ use bech32::Variant;
 use serde_derive::Deserialize;
 use serde_derive::Serialize;
 use twenty_first::math::b_field_element::BFieldElement;
+use twenty_first::math::bfield_codec::BFieldCodec;
 use twenty_first::math::lattice;
 use twenty_first::math::lattice::kem::CIPHERTEXT_SIZE_IN_BFES;
 use twenty_first::math::tip5::Digest;
+use zeroize::Zeroize;
 
 use super::common;
 use super::common::deterministically_derive_seed_and_nonce;
@@ -71,7 +73,7 @@ impl<'de> serde::de::Deserialize<'de> for GenerationSpendingKey {
         D: serde::de::Deserializer<'de>,
     {
         let seed = Digest::deserialize(deserializer)?;
-        Ok(Self::derive_from_seed(seed))
+        Ok(Self::from_seed(seed))
     }
 }
 
@@ -87,7 +89,22 @@ pub struct GenerationReceivingAddress {
 impl<'a> Arbitrary<'a> for GenerationReceivingAddress {
     fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
         let seed = Digest::arbitrary(u)?;
-        Ok(Self::derive_from_seed(seed))
+        Ok(Self::from_seed(seed))
+    }
+}
+
+impl Zeroize for GenerationSpendingKey {
+    // unsafe: we use unsafe std::mem::zeroed() to zero the
+    // SecretKey because it does not impl Zeroize or Default and
+    // it is expensive to create a new random key.
+    //
+    // It should be updated to impl Zeroize in twenty-first crate
+    fn zeroize(&mut self) {
+        self.receiver_identifier = Default::default();
+        self.decryption_key = unsafe { std::mem::zeroed() };
+        self.privacy_preimage = Default::default();
+        self.unlock_key = Default::default();
+        self.seed = Default::default();
     }
 }
 
@@ -108,7 +125,19 @@ impl GenerationSpendingKey {
         LockScriptAndWitness::hash_lock(self.unlock_key)
     }
 
-    pub fn derive_from_seed(seed: Digest) -> Self {
+    /// creates a GenerationSpendingKey from a seed
+    ///
+    /// security:
+    ///
+    /// It is critical that the seed is generated in a random fashion or itself
+    /// derived from a randomly generated seed.  Failure to do so can result in
+    /// a loss of funds.
+    ///
+    /// perf:
+    ///
+    /// not cheap. performs several hash ops, serialization, and lattice::kem
+    /// key-generation.
+    pub fn from_seed(seed: Digest) -> Self {
         let privacy_preimage =
             Hash::hash_varlen(&[seed.values().to_vec(), vec![BFieldElement::new(0)]].concat());
         let unlock_key =
@@ -125,20 +154,31 @@ impl GenerationSpendingKey {
             seed: seed.to_owned(),
         };
 
-        // Sanity check that spending key's receiver address can be encoded to
-        // bech32m without loss of information.
-        let receiving_address = spending_key.to_address();
-        let encoded_address = receiving_address.to_bech32m(Network::Alpha).unwrap();
-        let decoded_address =
-            GenerationReceivingAddress::from_bech32m(&encoded_address, Network::Alpha).unwrap();
-        assert_eq!(
-            receiving_address, decoded_address,
-            "encoding/decoding from bech32m must succeed. Receiving address was: {receiving_address:#?}"
-        );
+        #[cfg(debug_assertions)]
+        {
+            // Sanity check that spending key's receiver address can be encoded to
+            // bech32m without loss of information.
+            let receiving_address = spending_key.to_address();
+            let encoded_address = receiving_address.to_bech32m(Network::Alpha).unwrap();
+            let decoded_address =
+                GenerationReceivingAddress::from_bech32m(&encoded_address, Network::Alpha).unwrap();
+            assert_eq!(
+                receiving_address, decoded_address,
+                "encoding/decoding from bech32m must succeed. Receiving address was: {receiving_address:#?}"
+            );
+        }
 
         spending_key
     }
 
+    /// derives a child-key at index
+    ///
+    /// note that index 0 is the first child.
+    pub fn derive_child(&self, index: common::DerivationIndex) -> Self {
+        let child_seed = Hash::hash_varlen(&[self.seed.0.encode(), index.encode()].concat());
+        Self::from_seed(child_seed)
+    }
+
     /// Decrypt a Generation Address ciphertext
     pub(super) fn decrypt(&self, ciphertext: &[BFieldElement]) -> Result<(Utxo, Digest)> {
         // parse ciphertext
@@ -203,8 +243,8 @@ impl GenerationReceivingAddress {
         }
     }
 
-    pub fn derive_from_seed(seed: Digest) -> Self {
-        let spending_key = GenerationSpendingKey::derive_from_seed(seed);
+    pub fn from_seed(seed: Digest) -> Self {
+        let spending_key = GenerationSpendingKey::from_seed(seed);
         Self::from_spending_key(&spending_key)
     }