feat(engine)!: add Elgamal verifiable encryption (optional view key) …

…support (#976)

Description
---
feat(engine)!: add Elgamal verifiable encryption (optional view key)
support
feat: implement brute force value decryption for confidential outputs
using secret view key
refactor: move confidential wallet crypto into new crate
refactor(wallet/sdk): use wallet crypto crate
refactor(test tooling): use wallet crypto crate
tests(engine): new test checking confidential transfers with a view key
enabled
tests(wallet/crypto): checks that proof generation, verification and
decryption are valid
feat(engine): add simple interface for value lookup table

Motivation and Context
---
Allow a template author to specify a view key on confidential resources
optionally. This allows anyone with the secret key to uncover the
balance of commitments generated for the resource.

```rust
        let coins = ResourceBuilder::confidential()
                .initial_supply(confidential_proof)
                .with_view_key(view_key)
                .build_bucket();
```

Wallets MUST generate a ViewableBalanceProof for all confidential
outputs for the resource, allowing validators to verify that the
encrypted balance was generated correctly without revealing the balance.

All confidential crypto was duplicated in the test tooling and wallet
SDK. Since duplicating the Elgamal verifiable encryption scheme could
lead to issues down the road, or is just plain ugly, this PR puts all
confidential crypto into a crate that is used by the test tooling and
wallet SDK.

A value lookup table is passed into the brute force function. A
production implementation of may make use of the binary file provided by
the new generate_ristretto_value_lookup bin crate to return canonical
(compressed) bytes for a value. The implementation can optimise for
sequential reads and a low memory footprint. A binary file containing 1
billion entries (assuming 6 decimals, whole values from 0 - 1000) will
be 32 x 1B in size (32Gb).

A future PR will add wallet support for attempting to reveal the balance
of vaults.

How Has This Been Tested?
---
New unit tests

What process can a PR reviewer use to test or verify this change?
---
Create a template using the view key in a resource.
Check transfers work as before.
Fetch the commitments from a vault and reveal the balance (this is a
manual process currently without wallet support)

Breaking Changes
---
- [ ] None
- [ ] Requires data directory to be deleted
- [x] Other - Please specify

BREAKING CHANGES: added field to resource create args, meaning any
template using this would need to be recompiled.

Cargo.toml

@@ -40,6 +40,7 @@ members = [
     "dan_layer/transaction_manifest",
     "dan_layer/transaction",
     "dan_layer/validator_node_rpc",
+    "dan_layer/wallet/crypto",
     "dan_layer/wallet/sdk",
     "dan_layer/wallet/storage_sqlite",
     "integration_tests",
@@ -53,6 +54,7 @@ members = [
     "utilities/tariswap_test_bench",
     "utilities/transaction_submitter",
     "utilities/transaction_submitter",
+    "utilities/generate_ristretto_value_lookup",
 ]
 resolver = "2"
 
@@ -78,6 +80,7 @@ tari_dan_storage = { path = "dan_layer/storage" }
 tari_dan_storage_sqlite = { path = "dan_layer/storage_sqlite" }
 tari_dan_wallet_daemon = { path = "applications/tari_dan_wallet_daemon" }
 tari_dan_wallet_sdk = { path = "dan_layer/wallet/sdk" }
+tari_dan_wallet_crypto = { path = "dan_layer/wallet/crypto" }
 tari_dan_wallet_storage_sqlite = { path = "dan_layer/wallet/storage_sqlite" }
 tari_dan_p2p = { path = "dan_layer/p2p" }
 tari_engine_types = { path = "dan_layer/engine_types" }

applications/tari_dan_wallet_daemon/Cargo.toml

@@ -15,6 +15,7 @@ tari_crypto = { workspace = true }
 tari_common_types = { workspace = true }
 tari_dan_app_utilities = { workspace = true }
 tari_shutdown = { workspace = true }
+tari_dan_wallet_crypto = { workspace = true }
 tari_dan_wallet_sdk = { workspace = true }
 tari_dan_wallet_storage_sqlite = { workspace = true }
 tari_transaction = { workspace = true }

applications/tari_dan_wallet_daemon/src/handlers/accounts.rs

@@ -14,17 +14,17 @@ use tari_crypto::{
     tari_utilities::ByteArray,
 };
 use tari_dan_common_types::optional::Optional;
+use tari_dan_wallet_crypto::ConfidentialProofStatement;
 use tari_dan_wallet_sdk::{
     apis::{jwt::JrpcPermission, key_manager, substate::ValidatorScanResult},
-    confidential::{get_commitment_factory, ConfidentialProofStatement},
     models::{ConfidentialOutputModel, OutputStatus, VersionedSubstateId},
     storage::WalletStore,
     DanWalletSdk,
 };
 use tari_dan_wallet_storage_sqlite::SqliteWalletStore;
 use tari_engine_types::{
     component::new_account_address_from_parts,
-    confidential::ConfidentialClaim,
+    confidential::{get_commitment_factory, ConfidentialClaim},
     instruction::Instruction,
     substate::{Substate, SubstateId},
 };
@@ -387,6 +387,7 @@ pub async fn handle_reveal_funds(
             minimum_value_promise: 0,
             encrypted_data,
             reveal_amount: amount_to_reveal,
+            resource_view_key: None,
         };
 
         let inputs = sdk
@@ -623,6 +624,7 @@ pub async fn handle_claim_burn(
         minimum_value_promise: 0,
         encrypted_data,
         reveal_amount: max_fee,
+        resource_view_key: None,
     };
 
     let reveal_proof = sdk.confidential_crypto_api().generate_withdraw_proof(
@@ -1104,6 +1106,12 @@ pub async fn handle_confidential_transfer(
             .scan_for_substate(&SubstateId::Resource(req.resource_address), None)
             .await?;
         inputs.push(resource_substate.address);
+        let resource_view_key = resource_substate
+            .substate
+            .as_resource()
+            .ok_or_else(|| anyhow!("Indexer returned a non-resource substate when requesting a resource address"))?
+            .view_key()
+            .cloned();
 
         // get destination account information
         let destination_account_address =
@@ -1132,6 +1140,7 @@ pub async fn handle_confidential_transfer(
             encrypted_data,
             minimum_value_promise: 0,
             reveal_amount: Amount::zero(),
+            resource_view_key: resource_view_key.clone(),
         };
 
         let change_amount = total_input_value - req.amount.as_u64_checked().unwrap();
@@ -1168,6 +1177,7 @@ pub async fn handle_confidential_transfer(
                 minimum_value_promise: 0,
                 encrypted_data,
                 reveal_amount: Amount::zero(),
+                resource_view_key,
             })
         } else {
             None

applications/tari_dan_wallet_daemon/src/handlers/confidential.rs

@@ -9,11 +9,13 @@ use rand::rngs::OsRng;
 use serde_json::json;
 use tari_common_types::types::PublicKey;
 use tari_crypto::{commitment::HomomorphicCommitmentFactory, keys::PublicKey as _};
+use tari_dan_common_types::optional::Optional;
+use tari_dan_wallet_crypto::ConfidentialProofStatement;
 use tari_dan_wallet_sdk::{
     apis::{jwt::JrpcPermission, key_manager},
-    confidential::{get_commitment_factory, ConfidentialProofStatement},
     models::{ConfidentialOutputModel, OutputStatus},
 };
+use tari_engine_types::confidential::get_commitment_factory;
 use tari_template_lib::models::Amount;
 use tari_wallet_daemon_client::types::{
     ConfidentialCreateOutputProofRequest,
@@ -31,6 +33,7 @@ use crate::handlers::{
 
 const LOG_TARGET: &str = "tari::dan::wallet_daemon::json_rpc::confidential";
 
+#[allow(clippy::too_many_lines)]
 pub async fn handle_create_transfer_proof(
     context: &HandlerContext,
     token: Option<String>,
@@ -88,13 +91,34 @@ pub async fn handle_create_transfer_proof(
         &account_secret.key,
     )?;
 
+    let known_resource_substate_address = sdk
+        .substate_api()
+        .get_substate(&req.resource_address.into())
+        .optional()?;
+    let resource = sdk
+        .substate_api()
+        .scan_for_substate(
+            &req.resource_address.into(),
+            known_resource_substate_address.map(|s| s.address.version),
+        )
+        .await?;
+    let resource_view_key = resource
+        .substate
+        .as_resource()
+        .ok_or_else(|| {
+            anyhow::anyhow!("Indexer returned a non-resource substate when scanning for a resource address")
+        })?
+        .view_key()
+        .cloned();
+
     let output_statement = ConfidentialProofStatement {
         amount: req.amount,
         mask: output_mask.key,
         sender_public_nonce: public_nonce,
         minimum_value_promise: 0,
         encrypted_data,
         reveal_amount: req.reveal_amount,
+        resource_view_key: resource_view_key.clone(),
     };
 
     let change_amount = total_input_value - req.amount.value() as u64 - req.reveal_amount.value() as u64;
@@ -129,6 +153,7 @@ pub async fn handle_create_transfer_proof(
             encrypted_data,
             minimum_value_promise: 0,
             reveal_amount: Amount::zero(),
+            resource_view_key,
         })
     } else {
         None
@@ -200,6 +225,8 @@ pub async fn handle_create_output_proof(
         minimum_value_promise: 0,
         encrypted_data,
         reveal_amount: Amount::zero(),
+        // TODO: the request must include the resource address so that we can fetch the view key
+        resource_view_key: None,
     };
     let proof = sdk.confidential_crypto_api().generate_output_proof(&statement)?;
     Ok(ConfidentialCreateOutputProofResponse { proof })

applications/tari_indexer/src/json_rpc/json_encoding.rs

@@ -176,6 +176,7 @@ mod tests {
             stealth_public_nonce: commitment.as_public_key().clone(),
             encrypted_data: Default::default(),
             minimum_value_promise: 0,
+            viewable_balance: None,
         };
         let commitment = Some((commitment, confidential_output));
 

applications/tari_validator_node/src/bootstrap.rs

@@ -476,6 +476,7 @@ where
                 OwnerRule::None,
                 ResourceAccessRules::new(),
                 metadata,
+                None,
             )
             .into(),
             state_hash: Default::default(),
@@ -503,6 +504,7 @@ where
                 OwnerRule::None,
                 ResourceAccessRules::new(),
                 metadata,
+                None,
             )
             .into(),
             state_hash: Default::default(),

dan_layer/engine/src/bootstrap.rs

@@ -30,6 +30,7 @@ pub fn bootstrap_state<T: StateWriter>(state_db: &mut T) -> Result<(), StateStor
                 OwnerRule::None,
                 ResourceAccessRules::deny_all(),
                 metadata,
+                None,
             ),
         ),
     )?;
@@ -51,6 +52,7 @@ pub fn bootstrap_state<T: StateWriter>(state_db: &mut T) -> Result<(), StateStor
                     .withdrawable(AccessRule::AllowAll)
                     .depositable(AccessRule::AllowAll),
                 metadata,
+                None,
             ),
         ),
     )?;