From 3f89443846c543417b5f44ce0a1ab181b1ffd2c8 Mon Sep 17 00:00:00 2001 From: Philipp Gackstatter Date: Wed, 19 Feb 2025 20:37:05 +0100 Subject: [PATCH] feat: Update `get_block_inputs` for new `BlockInputs` --- Cargo.lock | 20 +- crates/block-producer/Cargo.toml | 1 + crates/block-producer/src/block.rs | 106 ----------- .../block-producer/src/block_builder/mod.rs | 114 +++++++----- crates/block-producer/src/lib.rs | 1 - crates/block-producer/src/store/mod.rs | 14 +- crates/proto/src/domain/account.rs | 30 ++- crates/proto/src/domain/block.rs | 107 ++++++++++- crates/proto/src/domain/nullifier.rs | 18 +- crates/proto/src/generated/requests.rs | 17 +- crates/proto/src/generated/responses.rs | 32 ++-- crates/proto/src/lib.rs | 4 +- crates/rpc-proto/proto/requests.proto | 16 +- crates/rpc-proto/proto/responses.proto | 24 +-- crates/store/src/errors.rs | 4 + crates/store/src/server/api.rs | 12 +- crates/store/src/state.rs | 175 +++++++++++------- proto/requests.proto | 16 +- proto/responses.proto | 24 +-- 19 files changed, 418 insertions(+), 317 deletions(-) delete mode 100644 crates/block-producer/src/block.rs diff --git a/Cargo.lock b/Cargo.lock index ae688eaca..e2ae127ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -837,7 +837,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -1703,6 +1703,17 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "miden-block-prover" +version = "0.8.0" +source = "git+https://github.com/0xPolygonMiden/miden-base?branch=pgackst-batch-expiration#fa7b5b1728f22d0c1b6264d62d96667002dbdb50" +dependencies = [ + "miden-crypto", + "miden-lib", + "miden-objects", + "thiserror 2.0.11", +] + [[package]] name = "miden-core" version = "0.12.0" @@ -1867,6 +1878,7 @@ dependencies = [ "futures", "itertools 0.14.0", "miden-air", + "miden-block-prover", "miden-lib", "miden-node-proto", "miden-node-test-macro", @@ -2918,7 +2930,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3322,7 +3334,7 @@ dependencies = [ "getrandom 0.3.1", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4131,7 +4143,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index 89a3f0cbc..2b904e44d 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -21,6 +21,7 @@ tracing-forest = ["miden-node-utils/tracing-forest"] async-trait = { version = "0.1" } futures = { version = "0.3" } itertools = { workspace = true } +miden-block-prover = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "pgackst-batch-expiration" } miden-lib = { workspace = true } miden-node-proto = { workspace = true } miden-node-utils = { workspace = true } diff --git a/crates/block-producer/src/block.rs b/crates/block-producer/src/block.rs deleted file mode 100644 index 857a5b018..000000000 --- a/crates/block-producer/src/block.rs +++ /dev/null @@ -1,106 +0,0 @@ -use std::collections::BTreeMap; - -use miden_node_proto::{ - domain::note::NoteAuthenticationInfo, - errors::{ConversionError, MissingFieldHelper}, - generated::responses::GetBlockInputsResponse, - AccountInputRecord, NullifierWitness, -}; -use miden_objects::{ - account::AccountId, - block::BlockHeader, - crypto::merkle::{MerklePath, MmrPeaks, SmtProof}, - note::Nullifier, - Digest, -}; - -// BLOCK INPUTS -// ================================================================================================ - -/// Information needed from the store to build a block -#[derive(Clone, Debug)] -pub struct BlockInputs { - /// Previous block header - pub block_header: BlockHeader, - - /// MMR peaks for the current chain state - pub chain_peaks: MmrPeaks, - - /// The hashes of the requested accounts and their authentication paths - pub accounts: BTreeMap, - - /// The requested nullifiers and their authentication paths - pub nullifiers: BTreeMap, - - /// List of unauthenticated notes found in the store - pub found_unauthenticated_notes: NoteAuthenticationInfo, -} - -#[derive(Clone, Debug, Default)] -pub struct AccountWitness { - pub hash: Digest, - pub proof: MerklePath, -} - -impl TryFrom for BlockInputs { - type Error = ConversionError; - - fn try_from(response: GetBlockInputsResponse) -> Result { - let block_header: BlockHeader = response - .block_header - .ok_or(miden_node_proto::generated::block::BlockHeader::missing_field("block_header"))? - .try_into()?; - - let chain_peaks = { - // setting the number of leaves to the current block number gives us one leaf less than - // what is currently in the chain MMR (i.e., chain MMR with block_num = 1 has 2 leave); - // this is because GetBlockInputs returns the state of the chain MMR as of one block - // ago so that block_header.chain_root matches the hash of MMR peaks. - let num_leaves = block_header.block_num().as_usize(); - - MmrPeaks::new( - num_leaves, - response - .mmr_peaks - .into_iter() - .map(TryInto::try_into) - .collect::>()?, - )? - }; - - let accounts = response - .account_states - .into_iter() - .map(|entry| { - let domain: AccountInputRecord = entry.try_into()?; - let witness = AccountWitness { - hash: domain.account_hash, - proof: domain.proof, - }; - Ok((domain.account_id, witness)) - }) - .collect::, ConversionError>>()?; - - let nullifiers = response - .nullifiers - .into_iter() - .map(|entry| { - let witness: NullifierWitness = entry.try_into()?; - Ok((witness.nullifier, witness.proof)) - }) - .collect::, ConversionError>>()?; - - let found_unauthenticated_notes = response - .found_unauthenticated_notes - .ok_or(GetBlockInputsResponse::missing_field("found_authenticated_notes"))? - .try_into()?; - - Ok(Self { - block_header, - chain_peaks, - accounts, - nullifiers, - found_unauthenticated_notes, - }) - } -} diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 738260312..4dd16dd0e 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -1,29 +1,27 @@ -use std::{ - collections::BTreeSet, - ops::{Add, Range}, -}; +use std::{collections::BTreeSet, ops::Range}; use futures::FutureExt; +use miden_block_prover::LocalBlockProver; use miden_node_utils::tracing::OpenTelemetrySpanExt; use miden_objects::{ account::AccountId, batch::ProvenBatch, - block::{BlockNumber, ProvenBlock}, + block::{BlockInputs, BlockNumber, ProposedBlock, ProvenBlock}, note::{NoteHeader, NoteId, Nullifier}, transaction::{InputNoteCommitment, OutputNote}, + MIN_PROOF_SECURITY_LEVEL, }; use rand::Rng; use tokio::time::Duration; use tracing::{instrument, Span}; use crate::{ - block::BlockInputs, errors::BuildBlockError, mempool::SharedMempool, store::StoreClient, - COMPONENT, SERVER_BLOCK_FREQUENCY, + errors::BuildBlockError, mempool::SharedMempool, store::StoreClient, COMPONENT, + SERVER_BLOCK_FREQUENCY, }; -pub(crate) mod prover; - -use self::prover::{block_witness::BlockWitness, BlockProver}; +// pub(crate) mod prover; +// use self::prover::BlockProver; // BLOCK BUILDER // ================================================================================================= @@ -39,7 +37,7 @@ pub struct BlockBuilder { pub failure_rate: f64, pub store: StoreClient, - pub block_kernel: BlockProver, + // pub block_prover: LocalBlockProver, } impl BlockBuilder { @@ -49,7 +47,7 @@ impl BlockBuilder { // Note: The range cannot be empty. simulated_proof_time: Duration::ZERO..Duration::from_millis(1), failure_rate: 0.0, - block_kernel: BlockProver::new(), + // block_prover: LocalBlockProver::new(MIN_PROOF_SECURITY_LEVEL), store, } } @@ -125,28 +123,52 @@ impl BlockBuilder { selected_block: SelectedBlock, ) -> Result { let SelectedBlock { block_number: _, batches } = selected_block; - let summary = BlockSummary::summarize_batches(&batches); + // let summary = BlockSummary::summarize_batches(&batches); + + // For a given set of batches, we need to get the following block inputs from the store: + // - note inclusion proofs for unauthenticated notes (not required to be complete due to the + // possibility of note erasure) + // - a chain MMR with: + // - all blocks referenced by batches + // - all blocks referenced by note inclusion proofs + // - account witnesses for all accounts updated in the block + // - nullifier witnesses for all nullifiers created in the block + // - since we don't yet know which nullifiers the block will actually create, we will + // supply a superset of all possible nullifiers the block might create, which are the + // nullifiers of all proven batches. However, if we knew that a certain note will be + // erased, we would not have to supply a nullifier witness for it. + + let batch_iter = batches.iter(); + + let unauthenticated_notes_iter = batch_iter.clone().flat_map(|batch| { + batch.input_notes().iter().filter_map(|note| note.header().map(NoteHeader::id)) + }); + let block_references_iter = batch_iter.clone().map(ProvenBatch::reference_block_num); + let account_ids = batch_iter.clone().flat_map(ProvenBatch::updated_accounts); + let created_nullifiers = batch_iter.flat_map(ProvenBatch::produced_nullifiers); let inputs = self .store .get_block_inputs( - summary.updated_accounts.iter().copied(), - summary.nullifiers.iter(), - summary.dangling_notes.iter(), + account_ids, + created_nullifiers, + unauthenticated_notes_iter, + block_references_iter, ) .await .map_err(BuildBlockError::GetBlockInputsFailed)?; - let missing_notes: Vec<_> = summary - .dangling_notes - .difference(&inputs.found_unauthenticated_notes.note_ids()) - .copied() - .collect(); - if !missing_notes.is_empty() { - return Err(BuildBlockError::UnauthenticatedNotesNotFound(missing_notes)); - } - - Ok(BlockSummaryAndInputs { batches, summary, inputs }) + // No longer needed, we have note erasure at block level. + // let missing_notes: Vec<_> = summary + // .dangling_notes + // .difference(&inputs.found_unauthenticated_notes.note_ids()) + // .copied() + // .collect(); + // if !missing_notes.is_empty() { + // return Err(BuildBlockError::UnauthenticatedNotesNotFound(missing_notes)); + // } + + Ok(BlockSummaryAndInputs { batches, inputs }) } #[instrument(target = COMPONENT, name = "block_builder.prove_block", skip_all, err)] @@ -154,23 +176,17 @@ impl BlockBuilder { &self, preimage: BlockSummaryAndInputs, ) -> Result { - let BlockSummaryAndInputs { batches, summary, inputs } = preimage; + let BlockSummaryAndInputs { batches, inputs } = preimage; - let (block_header_witness, updated_accounts) = BlockWitness::new(inputs, &batches)?; + let proposed_block = ProposedBlock::new(inputs, batches).expect("TODO: error"); - let new_block_header = self.block_kernel.prove(block_header_witness)?; - - // TODO: Update. Temporarily left in an incorrect state. - let block = ProvenBlock::new_unchecked( - new_block_header, - updated_accounts, - vec![], - summary.nullifiers, - ); + let proven_block = LocalBlockProver::new(MIN_PROOF_SECURITY_LEVEL) + .prove(proposed_block) + .expect("TODO: error"); self.simulate_proving().await; - Ok(ProvenBlockWrapper { block }) + Ok(ProvenBlockWrapper { block: proven_block }) } #[instrument(target = COMPONENT, name = "block_builder.commit_block", skip_all, err)] @@ -273,7 +289,6 @@ struct SelectedBlock { } struct BlockSummaryAndInputs { batches: Vec, - summary: BlockSummary, inputs: BlockInputs, } @@ -297,21 +312,17 @@ impl BlockSummaryAndInputs { // SAFETY: We do not expect to have more than u32::MAX of any count per block. span.set_attribute( "block.updated_accounts.count", - i64::try_from(self.summary.updated_accounts.len()) + i64::try_from(self.inputs.account_witnesses().len()) .expect("less than u32::MAX account updates"), ); - span.set_attribute( - "block.output_notes.count", - i64::try_from(self.summary.output_notes.iter().fold(0, |acc, x| acc.add(x.len()))) - .expect("less than u32::MAX output notes"), - ); span.set_attribute( "block.nullifiers.count", - i64::try_from(self.summary.nullifiers.len()).expect("less than u32::MAX nullifiers"), + i64::try_from(self.inputs.nullifier_witnesses().len()) + .expect("less than u32::MAX nullifiers"), ); span.set_attribute( - "block.dangling_notes.count", - i64::try_from(self.summary.dangling_notes.len()) + "block.unauthenticated_notes.count", + i64::try_from(self.inputs.unauthenticated_note_proofs().len()) .expect("less than u32::MAX dangling notes"), ); } @@ -328,6 +339,13 @@ impl ProvenBlockWrapper { span.set_attribute("block.protocol.version", i64::from(header.version())); + // Question: Should this be here? + span.set_attribute( + "block.output_notes.count", + i64::try_from(self.block.output_notes().count()) + .expect("less than u32::MAX output notes"), + ); + span.set_attribute("block.commitments.kernel", header.kernel_root()); span.set_attribute("block.commitments.nullifier", header.nullifier_root()); span.set_attribute("block.commitments.account", header.account_root()); diff --git a/crates/block-producer/src/lib.rs b/crates/block-producer/src/lib.rs index 1cb3b62c9..8aa594232 100644 --- a/crates/block-producer/src/lib.rs +++ b/crates/block-producer/src/lib.rs @@ -10,7 +10,6 @@ mod errors; mod mempool; mod store; -pub mod block; pub mod config; pub mod server; diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index d367a7a75..9d4ad4d40 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -22,7 +22,7 @@ use miden_node_proto::{ use miden_node_utils::{formatting::format_opt, tracing::grpc::OtelInterceptor}; use miden_objects::{ account::AccountId, - block::{BlockHeader, BlockNumber, ProvenBlock}, + block::{BlockHeader, BlockInputs, BlockNumber, ProvenBlock}, note::{NoteId, Nullifier}, transaction::ProvenTransaction, utils::Serializable, @@ -32,7 +32,7 @@ use miden_processor::crypto::RpoDigest; use tonic::{service::interceptor::InterceptedService, transport::Channel}; use tracing::{debug, info, instrument}; -use crate::{block::BlockInputs, errors::StoreError, COMPONENT}; +use crate::{errors::StoreError, COMPONENT}; // TRANSACTION INPUTS // ================================================================================================ @@ -197,13 +197,15 @@ impl StoreClient { pub async fn get_block_inputs( &self, updated_accounts: impl Iterator + Send, - produced_nullifiers: impl Iterator + Send, - notes: impl Iterator + Send, + created_nullifiers: impl Iterator + Send, + unauthenticated_notes: impl Iterator + Send, + reference_blocks: impl Iterator + Send, ) -> Result { let request = tonic::Request::new(GetBlockInputsRequest { account_ids: updated_accounts.map(Into::into).collect(), - nullifiers: produced_nullifiers.map(digest::Digest::from).collect(), - unauthenticated_notes: notes.map(digest::Digest::from).collect(), + nullifiers: created_nullifiers.map(digest::Digest::from).collect(), + unauthenticated_notes: unauthenticated_notes.map(digest::Digest::from).collect(), + reference_blocks: reference_blocks.map(|block_num| block_num.as_u32()).collect(), }); let store_response = self.inner.clone().get_block_inputs(request).await?.into_inner(); diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 3d6680065..5a8775ace 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -3,7 +3,7 @@ use std::fmt::{Debug, Display, Formatter}; use miden_node_utils::formatting::format_opt; use miden_objects::{ account::{Account, AccountHeader, AccountId}, - block::BlockNumber, + block::{AccountWitness, BlockNumber}, crypto::{hash::rpo::RpoDigest, merkle::MerklePath}, utils::{Deserializable, Serializable}, Digest, @@ -153,45 +153,41 @@ impl TryInto for proto::requests::get_account_proofs_reques // ================================================================================================ #[derive(Clone, Debug)] -pub struct AccountInputRecord { +pub struct AccountWitnessRecord { pub account_id: AccountId, - pub account_hash: Digest, + pub initial_state_commitment: Digest, pub proof: MerklePath, } -impl From for proto::responses::AccountBlockInputRecord { - fn from(from: AccountInputRecord) -> Self { +impl From for proto::responses::AccountWitness { + fn from(from: AccountWitnessRecord) -> Self { Self { account_id: Some(from.account_id.into()), - account_hash: Some(from.account_hash.into()), + initial_state_commitment: Some(from.initial_state_commitment.into()), proof: Some(Into::into(&from.proof)), } } } -impl TryFrom for AccountInputRecord { +impl TryFrom for AccountWitnessRecord { type Error = ConversionError; fn try_from( - account_input_record: proto::responses::AccountBlockInputRecord, + account_input_record: proto::responses::AccountWitness, ) -> Result { Ok(Self { account_id: account_input_record .account_id - .ok_or(proto::responses::AccountBlockInputRecord::missing_field(stringify!( - account_id - )))? + .ok_or(proto::responses::AccountWitness::missing_field(stringify!(account_id)))? .try_into()?, - account_hash: account_input_record - .account_hash - .ok_or(proto::responses::AccountBlockInputRecord::missing_field(stringify!( - account_hash - )))? + initial_state_commitment: account_input_record + .initial_state_commitment + .ok_or(proto::responses::AccountWitness::missing_field(stringify!(account_hash)))? .try_into()?, proof: account_input_record .proof .as_ref() - .ok_or(proto::responses::AccountBlockInputRecord::missing_field(stringify!(proof)))? + .ok_or(proto::responses::AccountWitness::missing_field(stringify!(proof)))? .try_into()?, }) } diff --git a/crates/proto/src/domain/block.rs b/crates/proto/src/domain/block.rs index fa7e4bcfb..43b7f2f1a 100644 --- a/crates/proto/src/domain/block.rs +++ b/crates/proto/src/domain/block.rs @@ -1,11 +1,19 @@ +use std::collections::BTreeMap; + use miden_objects::{ - block::{BlockHeader, BlockNumber}, + block::{AccountWitness, BlockHeader, BlockInputs, BlockNumber, NullifierWitness}, crypto::merkle::MerklePath, + note::{NoteId, NoteInclusionProof}, + transaction::ChainMmr, + utils::{Deserializable, Serializable}, }; use crate::{ errors::{ConversionError, MissingFieldHelper}, - generated::block as proto, + generated::{ + block as proto, note::NoteInclusionInBlockProof, responses::GetBlockInputsResponse, + }, + AccountWitnessRecord, NullifierWitnessRecord, }; // BLOCK HEADER @@ -124,3 +132,98 @@ impl TryFrom for BlockInclusionProof { Ok(result) } } + +// BLOCK INPUTS +// ================================================================================================ + +impl From for GetBlockInputsResponse { + fn from(inputs: BlockInputs) -> Self { + let ( + prev_block_header, + chain_mmr, + account_witnesses, + nullifier_witnesses, + unauthenticated_note_proofs, + ) = inputs.into_parts(); + + GetBlockInputsResponse { + latest_block_header: Some(prev_block_header.into()), + account_witnesses: account_witnesses + .into_iter() + .map(|(id, witness)| { + let (initial_state_commitment, proof) = witness.into_parts(); + AccountWitnessRecord { + account_id: id, + initial_state_commitment, + proof, + } + .into() + }) + .collect(), + nullifier_witnesses: nullifier_witnesses + .into_iter() + .map(|(nullifier, witness)| { + let proof = witness.into_proof(); + NullifierWitnessRecord { nullifier, proof }.into() + }) + .collect(), + chain_mmr: chain_mmr.to_bytes(), + unauthenticated_note_proofs: unauthenticated_note_proofs + .iter() + .map(NoteInclusionInBlockProof::from) + .collect(), + } + } +} + +impl TryFrom for BlockInputs { + type Error = ConversionError; + + fn try_from(response: GetBlockInputsResponse) -> Result { + let latest_block_header: BlockHeader = response + .latest_block_header + .ok_or(proto::BlockHeader::missing_field("block_header"))? + .try_into()?; + + let account_witnesses = response + .account_witnesses + .into_iter() + .map(|entry| { + let witness_record: AccountWitnessRecord = entry.try_into()?; + Ok(( + witness_record.account_id, + AccountWitness::new( + witness_record.initial_state_commitment, + witness_record.proof, + ), + )) + }) + .collect::, ConversionError>>()?; + + let nullifier_witnesses = response + .nullifier_witnesses + .into_iter() + .map(|entry| { + let witness: NullifierWitnessRecord = entry.try_into()?; + Ok((witness.nullifier, NullifierWitness::new(witness.proof))) + }) + .collect::, ConversionError>>()?; + + let unauthenticated_note_proofs = response + .unauthenticated_note_proofs + .iter() + .map(<(NoteId, NoteInclusionProof)>::try_from) + .collect::>()?; + + let chain_mmr = ChainMmr::read_from_bytes(&response.chain_mmr) + .map_err(|source| ConversionError::deserialization_error("ChainMmr", source))?; + + Ok(BlockInputs::new( + latest_block_header, + chain_mmr, + account_witnesses, + nullifier_witnesses, + unauthenticated_note_proofs, + )) + } +} diff --git a/crates/proto/src/domain/nullifier.rs b/crates/proto/src/domain/nullifier.rs index 482183a0f..548889d42 100644 --- a/crates/proto/src/domain/nullifier.rs +++ b/crates/proto/src/domain/nullifier.rs @@ -39,36 +39,32 @@ impl TryFrom for Nullifier { // ================================================================================================ #[derive(Clone, Debug)] -pub struct NullifierWitness { +pub struct NullifierWitnessRecord { pub nullifier: Nullifier, pub proof: SmtProof, } -impl TryFrom for NullifierWitness { +impl TryFrom for NullifierWitnessRecord { type Error = ConversionError; fn try_from( - nullifier_input_record: proto::responses::NullifierBlockInputRecord, + nullifier_input_record: proto::responses::NullifierWitness, ) -> Result { Ok(Self { nullifier: nullifier_input_record .nullifier - .ok_or(proto::responses::NullifierBlockInputRecord::missing_field(stringify!( - nullifier - )))? + .ok_or(proto::responses::NullifierWitness::missing_field(stringify!(nullifier)))? .try_into()?, proof: nullifier_input_record .opening - .ok_or(proto::responses::NullifierBlockInputRecord::missing_field(stringify!( - opening - )))? + .ok_or(proto::responses::NullifierWitness::missing_field(stringify!(opening)))? .try_into()?, }) } } -impl From for proto::responses::NullifierBlockInputRecord { - fn from(value: NullifierWitness) -> Self { +impl From for proto::responses::NullifierWitness { + fn from(value: NullifierWitnessRecord) -> Self { Self { nullifier: Some(value.nullifier.into()), opening: Some(value.proof.into()), diff --git a/crates/proto/src/generated/requests.rs b/crates/proto/src/generated/requests.rs index c8e19bb29..ac94d9ff7 100644 --- a/crates/proto/src/generated/requests.rs +++ b/crates/proto/src/generated/requests.rs @@ -82,15 +82,26 @@ pub struct SyncNoteRequest { /// Returns data required to prove the next block. #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetBlockInputsRequest { - /// ID of the account against which a transaction is executed. + /// IDs of all accounts updated in the proposed block for which to retrieve account witnesses. #[prost(message, repeated, tag = "1")] pub account_ids: ::prost::alloc::vec::Vec, - /// Set of nullifiers consumed by this transaction. + /// Nullifiers of all notes that will be consumed by the block. + /// + /// Due to note erasure it will generally not be possible to know the exact set of nullifiers + /// a block will create, unless we pre-execute note erasure. So in practice, this set of + /// nullifiers will be the set of nullifiers of all proven batches in the block, which is a + /// superset of the nullifiers the block may create. + /// + /// However, if it is known that a certain note will be erased, it would not be necessary to + /// provide a nullifier witness for it. #[prost(message, repeated, tag = "2")] pub nullifiers: ::prost::alloc::vec::Vec, - /// Array of note IDs to be checked for existence in the database. + /// Array of note IDs for which to retrieve note inclusion proofs. #[prost(message, repeated, tag = "3")] pub unauthenticated_notes: ::prost::alloc::vec::Vec, + /// Array of blocks referenced by all batches in the block. + #[prost(uint32, repeated, tag = "4")] + pub reference_blocks: ::prost::alloc::vec::Vec, } /// Returns the inputs for a transaction batch. #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/crates/proto/src/generated/responses.rs b/crates/proto/src/generated/responses.rs index c3a8f5f20..87b4f4d1c 100644 --- a/crates/proto/src/generated/responses.rs +++ b/crates/proto/src/generated/responses.rs @@ -86,20 +86,20 @@ pub struct SyncNoteResponse { } /// An account returned as a response to the `GetBlockInputs`. #[derive(Clone, PartialEq, ::prost::Message)] -pub struct AccountBlockInputRecord { +pub struct AccountWitness { /// The account ID. #[prost(message, optional, tag = "1")] pub account_id: ::core::option::Option, /// The latest account hash, zero hash if the account doesn't exist. #[prost(message, optional, tag = "2")] - pub account_hash: ::core::option::Option, + pub initial_state_commitment: ::core::option::Option, /// Merkle path to verify the account's inclusion in the MMR. #[prost(message, optional, tag = "3")] pub proof: ::core::option::Option, } /// A nullifier returned as a response to the `GetBlockInputs`. #[derive(Clone, PartialEq, ::prost::Message)] -pub struct NullifierBlockInputRecord { +pub struct NullifierWitness { /// The nullifier ID. #[prost(message, optional, tag = "1")] pub nullifier: ::core::option::Option, @@ -112,21 +112,23 @@ pub struct NullifierBlockInputRecord { pub struct GetBlockInputsResponse { /// The latest block header. #[prost(message, optional, tag = "1")] - pub block_header: ::core::option::Option, - /// Peaks of the above block's mmr, The `forest` value is equal to the block number. + pub latest_block_header: ::core::option::Option, + /// Proof of each _found_ unauthenticated note's inclusion in a block. #[prost(message, repeated, tag = "2")] - pub mmr_peaks: ::prost::alloc::vec::Vec, + pub unauthenticated_note_proofs: ::prost::alloc::vec::Vec< + super::note::NoteInclusionInBlockProof, + >, + /// The serialized chain MMR which includes proofs for all blocks referenced by the + /// above note inclusion proofs as well as proofs for inclusion of the blocks referenced + /// by the batches in the block. + #[prost(bytes = "vec", tag = "3")] + pub chain_mmr: ::prost::alloc::vec::Vec, /// The hashes of the requested accounts and their authentication paths. - #[prost(message, repeated, tag = "3")] - pub account_states: ::prost::alloc::vec::Vec, - /// The requested nullifiers and their authentication paths. #[prost(message, repeated, tag = "4")] - pub nullifiers: ::prost::alloc::vec::Vec, - /// The list of requested notes which were found in the database. - #[prost(message, optional, tag = "5")] - pub found_unauthenticated_notes: ::core::option::Option< - super::note::NoteAuthenticationInfo, - >, + pub account_witnesses: ::prost::alloc::vec::Vec, + /// The requested nullifiers and their authentication paths. + #[prost(message, repeated, tag = "5")] + pub nullifier_witnesses: ::prost::alloc::vec::Vec, } /// Represents the result of getting batch inputs. #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/crates/proto/src/lib.rs b/crates/proto/src/lib.rs index 9290fc739..ca950f020 100644 --- a/crates/proto/src/lib.rs +++ b/crates/proto/src/lib.rs @@ -8,8 +8,8 @@ pub mod generated; // ================================================================================================ pub use domain::{ - account::{AccountInputRecord, AccountState}, + account::{AccountState, AccountWitnessRecord}, convert, - nullifier::NullifierWitness, + nullifier::NullifierWitnessRecord, try_convert, }; diff --git a/crates/rpc-proto/proto/requests.proto b/crates/rpc-proto/proto/requests.proto index f2323c56c..d87726604 100644 --- a/crates/rpc-proto/proto/requests.proto +++ b/crates/rpc-proto/proto/requests.proto @@ -78,12 +78,22 @@ message SyncNoteRequest { // Returns data required to prove the next block. message GetBlockInputsRequest { - // ID of the account against which a transaction is executed. + // IDs of all accounts updated in the proposed block for which to retrieve account witnesses. repeated account.AccountId account_ids = 1; - // Set of nullifiers consumed by this transaction. + // Nullifiers of all notes that will be consumed by the block. + // + // Due to note erasure it will generally not be possible to know the exact set of nullifiers + // a block will create, unless we pre-execute note erasure. So in practice, this set of + // nullifiers will be the set of nullifiers of all proven batches in the block, which is a + // superset of the nullifiers the block may create. + // + // However, if it is known that a certain note will be erased, it would not be necessary to + // provide a nullifier witness for it. repeated digest.Digest nullifiers = 2; - // Array of note IDs to be checked for existence in the database. + // Array of note IDs for which to retrieve note inclusion proofs. repeated digest.Digest unauthenticated_notes = 3; + // Array of blocks referenced by all batches in the block. + repeated uint32 reference_blocks = 4; } // Returns the inputs for a transaction batch. diff --git a/crates/rpc-proto/proto/responses.proto b/crates/rpc-proto/proto/responses.proto index f1dfe5f90..771ed52bc 100644 --- a/crates/rpc-proto/proto/responses.proto +++ b/crates/rpc-proto/proto/responses.proto @@ -90,19 +90,19 @@ message SyncNoteResponse { } // An account returned as a response to the `GetBlockInputs`. -message AccountBlockInputRecord { +message AccountWitness { // The account ID. account.AccountId account_id = 1; // The latest account hash, zero hash if the account doesn't exist. - digest.Digest account_hash = 2; + digest.Digest initial_state_commitment = 2; // Merkle path to verify the account's inclusion in the MMR. merkle.MerklePath proof = 3; } // A nullifier returned as a response to the `GetBlockInputs`. -message NullifierBlockInputRecord { +message NullifierWitness { // The nullifier ID. digest.Digest nullifier = 1; @@ -113,19 +113,21 @@ message NullifierBlockInputRecord { // Represents the result of getting block inputs. message GetBlockInputsResponse { // The latest block header. - block.BlockHeader block_header = 1; + block.BlockHeader latest_block_header = 1; - // Peaks of the above block's mmr, The `forest` value is equal to the block number. - repeated digest.Digest mmr_peaks = 2; + // Proof of each _found_ unauthenticated note's inclusion in a block. + repeated note.NoteInclusionInBlockProof unauthenticated_note_proofs = 2; + + // The serialized chain MMR which includes proofs for all blocks referenced by the + // above note inclusion proofs as well as proofs for inclusion of the blocks referenced + // by the batches in the block. + bytes chain_mmr = 3; // The hashes of the requested accounts and their authentication paths. - repeated AccountBlockInputRecord account_states = 3; + repeated AccountWitness account_witnesses = 4; // The requested nullifiers and their authentication paths. - repeated NullifierBlockInputRecord nullifiers = 4; - - // The list of requested notes which were found in the database. - note.NoteAuthenticationInfo found_unauthenticated_notes = 5; + repeated NullifierWitness nullifier_witnesses = 5; } // Represents the result of getting batch inputs. diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index aecf56f33..d9ba0110b 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -244,6 +244,10 @@ pub enum GetBlockInputsError { IncorrectChainMmrForestNumber { forest: usize, block_num: BlockNumber }, #[error("note inclusion proof MMR error")] NoteInclusionMmr(#[from] MmrError), + #[error("failed to select note inclusion proofs")] + SelectNoteInclusionProofError(#[source] DatabaseError), + #[error("failed to select block headers")] + SelectBlockHeaderError(#[source] DatabaseError), } impl From for GetBlockInputsError { diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 0064668e6..a509e1723 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -349,15 +349,16 @@ impl api_server::Api for StoreApi { ) -> Result, Status> { let request = request.into_inner(); - let nullifiers = validate_nullifiers(&request.nullifiers)?; let account_ids = read_account_ids(&request.account_ids)?; + let nullifiers = validate_nullifiers(&request.nullifiers)?; let unauthenticated_notes = validate_notes(&request.unauthenticated_notes)?; + let reference_blocks = read_block_numbers(&request.reference_blocks); let unauthenticated_notes = unauthenticated_notes.into_iter().collect(); self.state - .get_block_inputs(&account_ids, &nullifiers, unauthenticated_notes) + .get_block_inputs(&account_ids, &nullifiers, unauthenticated_notes, reference_blocks) .await - .map(Into::into) + .map(GetBlockInputsResponse::from) .map(Response::new) .map_err(internal_error) } @@ -580,3 +581,8 @@ fn validate_notes(notes: &[generated::digest::Digest]) -> Result, St .collect::>() .map_err(|_| invalid_argument("Digest field is not in the modulus range")) } + +#[instrument(target = COMPONENT, skip_all)] +fn read_block_numbers(block_numbers: &[u32]) -> BTreeSet { + block_numbers.iter().map(|raw_number| BlockNumber::from(*raw_number)).collect() +} diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 17ec2cf45..f8865fa90 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -20,12 +20,12 @@ use miden_node_proto::{ generated::responses::{ AccountProofsResponse, AccountStateHeader, GetBlockInputsResponse, StorageSlotMapProof, }, - AccountInputRecord, NullifierWitness, + AccountWitnessRecord, }; use miden_node_utils::formatting::format_array; use miden_objects::{ account::{AccountDelta, AccountHeader, AccountId, StorageSlot}, - block::{BlockHeader, BlockNumber, ProvenBlock}, + block::{AccountWitness, BlockHeader, BlockInputs, BlockNumber, NullifierWitness, ProvenBlock}, crypto::{ hash::rpo::RpoDigest, merkle::{ @@ -58,36 +58,24 @@ use crate::{ // STRUCTURES // ================================================================================================ -/// Information needed from the store to validate and build a block -#[derive(Debug)] -pub struct BlockInputs { - /// Previous block header - pub block_header: BlockHeader, - - /// MMR peaks for the current chain state - pub chain_peaks: MmrPeaks, +// /// Information needed from the store to validate and build a block +// #[derive(Debug)] +// pub struct BlockInputs { +// /// Previous block header +// pub block_header: BlockHeader, - /// The hashes of the requested accounts and their authentication paths - pub account_states: Vec, +// /// MMR peaks for the current chain state +// pub chain_peaks: MmrPeaks, - /// The requested nullifiers and their authentication paths - pub nullifiers: Vec, +// /// The hashes of the requested accounts and their authentication paths +// pub account_states: Vec, - /// List of notes found in the store - pub found_unauthenticated_notes: NoteAuthenticationInfo, -} +// /// The requested nullifiers and their authentication paths +// pub nullifiers: Vec, -impl From for GetBlockInputsResponse { - fn from(value: BlockInputs) -> Self { - Self { - block_header: Some(value.block_header.into()), - mmr_peaks: convert(value.chain_peaks.peaks()), - account_states: convert(value.account_states), - nullifiers: convert(value.nullifiers), - found_unauthenticated_notes: Some(value.found_unauthenticated_notes.into()), - } - } -} +// /// List of notes found in the store +// pub found_unauthenticated_notes: NoteAuthenticationInfo, +// } #[derive(Debug)] pub struct TransactionInputs { @@ -155,7 +143,7 @@ impl Blockchain { &self, blocks: &BTreeSet, latest_block_number: BlockNumber, - ) -> Result { + ) -> PartialMmr { // Using latest block as the target forest means we take the state of the MMR one before // the latest block. This is because the latest block will be used as the reference // block of the batch and will be added to the MMR by the batch kernel. @@ -183,7 +171,8 @@ impl Blockchain { .track(block_num, leaf, &path) .expect("filling partial mmr with data from mmr should succeed"); } - Ok(partial_mmr) + + partial_mmr } } @@ -654,7 +643,7 @@ impl State { ( latest_block_num, - inner_state.blockchain.partial_mmr_from_blocks(&blocks, latest_block_num)?, + inner_state.blockchain.partial_mmr_from_blocks(&blocks, latest_block_num), ) }; @@ -783,61 +772,105 @@ impl State { account_ids: &[AccountId], nullifiers: &[Nullifier], unauthenticated_notes: BTreeSet, + mut reference_blocks: BTreeSet, ) -> Result { + let unauthenticated_note_proofs = self + .db + .select_note_inclusion_proofs(unauthenticated_notes) + .await + .map_err(GetBlockInputsError::SelectNoteInclusionProofError)?; + + // The set of blocks that the notes are included in. + let note_proof_reference_blocks = + unauthenticated_note_proofs.values().map(|proof| proof.location().block_num()); + + reference_blocks.extend(note_proof_reference_blocks); + let inner = self.inner.read().await; - let latest = self - .db - .select_block_header_by_block_num(None) - .await? - .ok_or(GetBlockInputsError::DbBlockHeaderEmpty)?; - - // sanity check - if inner.blockchain.chain_tip() != latest.block_num() { - return Err(GetBlockInputsError::IncorrectChainMmrForestNumber { - forest: inner.blockchain.chain_tip().as_usize(), - block_num: latest.block_num(), - }); - } + let latest_block_number = inner.latest_block_num(); - // using current block number gets us the peaks of the chain MMR as of one block ago; - // this is done so that latest.chain_root matches the returned peaks - let chain_peaks = - inner.blockchain.peaks_at(latest.block_num().as_usize()).map_err(|error| { - GetBlockInputsError::FailedToGetMmrPeaksForForest { - forest: latest.block_num().as_usize(), - error, - } - })?; - let account_states = account_ids + reference_blocks.remove(&latest_block_number); + + let partial_mmr = + inner.blockchain.partial_mmr_from_blocks(&reference_blocks, latest_block_number); + + let account_witnesses = account_ids .iter() .copied() .map(|account_id| { - let ValuePath { value: account_hash, path: proof } = - inner.account_tree.open(&LeafIndex::new_max_depth(account_id.prefix().into())); - Ok(AccountInputRecord { account_id, account_hash, proof }) + let ValuePath { + value: initial_state_commitment, + path: proof, + } = inner.account_tree.open(&account_id.into()); + (account_id, AccountWitness::new(initial_state_commitment, proof)) }) - .collect::>()?; + .collect::>(); - let nullifiers: Vec = nullifiers + let nullifier_witnesses: BTreeMap = nullifiers .iter() + .copied() .map(|nullifier| { - let proof = inner.nullifier_tree.open(nullifier); - - NullifierWitness { nullifier: *nullifier, proof } + let proof = inner.nullifier_tree.open(&nullifier); + (nullifier, NullifierWitness::new(proof)) }) .collect(); - let found_unauthenticated_notes = - self.get_note_authentication_info(unauthenticated_notes).await?; + // Release the lock. + std::mem::drop(inner); - Ok(BlockInputs { - block_header: latest, - chain_peaks, - account_states, - nullifiers, - found_unauthenticated_notes, - }) + let mut headers = self + .db + .select_block_headers( + reference_blocks.into_iter().chain(std::iter::once(latest_block_number)), + ) + .await + .map_err(GetBlockInputsError::SelectBlockHeaderError)?; + + // Find and remove the latest block as we must not add it to the chain MMR, since it is + // not yet in the chain. + let latest_block_header_index = headers + .iter() + .enumerate() + .find_map(|(index, header)| { + (header.block_num() == latest_block_number).then_some(index) + }) + .expect("DB should have returned the header of the latest block header"); + + // The order doesn't matter for ChainMmr::new, so swap remove is fine. + let latest_block_header = headers.swap_remove(latest_block_header_index); + + // SAFETY: This should not error because: + // - we're passing exactly the block headers that we've added to the partial MMR, + // - so none of the block header's block numbers should exceed the chain length of the + // partial MMR, + // - and we've added blocks to a BTreeSet, so there can be no duplicates. + let chain_mmr = ChainMmr::new(partial_mmr, headers) + .expect("partial mmr and block headers should be consistent"); + + Ok(BlockInputs::new( + latest_block_header, + chain_mmr, + account_witnesses, + nullifier_witnesses, + unauthenticated_note_proofs, + )) + + // // sanity check + // if inner.blockchain.chain_tip() != latest.block_num() { + // return Err(GetBlockInputsError::IncorrectChainMmrForestNumber { + // forest: inner.blockchain.chain_tip().as_usize(), + // block_num: latest.block_num(), + // }); + // } + + // Ok(BlockInputs { + // block_header: latest, + // chain_peaks, + // account_states, + // nullifiers, + // found_unauthenticated_notes, + // }) } /// Returns data needed by the block producer to verify transactions validity. diff --git a/proto/requests.proto b/proto/requests.proto index f2323c56c..d87726604 100644 --- a/proto/requests.proto +++ b/proto/requests.proto @@ -78,12 +78,22 @@ message SyncNoteRequest { // Returns data required to prove the next block. message GetBlockInputsRequest { - // ID of the account against which a transaction is executed. + // IDs of all accounts updated in the proposed block for which to retrieve account witnesses. repeated account.AccountId account_ids = 1; - // Set of nullifiers consumed by this transaction. + // Nullifiers of all notes that will be consumed by the block. + // + // Due to note erasure it will generally not be possible to know the exact set of nullifiers + // a block will create, unless we pre-execute note erasure. So in practice, this set of + // nullifiers will be the set of nullifiers of all proven batches in the block, which is a + // superset of the nullifiers the block may create. + // + // However, if it is known that a certain note will be erased, it would not be necessary to + // provide a nullifier witness for it. repeated digest.Digest nullifiers = 2; - // Array of note IDs to be checked for existence in the database. + // Array of note IDs for which to retrieve note inclusion proofs. repeated digest.Digest unauthenticated_notes = 3; + // Array of blocks referenced by all batches in the block. + repeated uint32 reference_blocks = 4; } // Returns the inputs for a transaction batch. diff --git a/proto/responses.proto b/proto/responses.proto index f1dfe5f90..771ed52bc 100644 --- a/proto/responses.proto +++ b/proto/responses.proto @@ -90,19 +90,19 @@ message SyncNoteResponse { } // An account returned as a response to the `GetBlockInputs`. -message AccountBlockInputRecord { +message AccountWitness { // The account ID. account.AccountId account_id = 1; // The latest account hash, zero hash if the account doesn't exist. - digest.Digest account_hash = 2; + digest.Digest initial_state_commitment = 2; // Merkle path to verify the account's inclusion in the MMR. merkle.MerklePath proof = 3; } // A nullifier returned as a response to the `GetBlockInputs`. -message NullifierBlockInputRecord { +message NullifierWitness { // The nullifier ID. digest.Digest nullifier = 1; @@ -113,19 +113,21 @@ message NullifierBlockInputRecord { // Represents the result of getting block inputs. message GetBlockInputsResponse { // The latest block header. - block.BlockHeader block_header = 1; + block.BlockHeader latest_block_header = 1; - // Peaks of the above block's mmr, The `forest` value is equal to the block number. - repeated digest.Digest mmr_peaks = 2; + // Proof of each _found_ unauthenticated note's inclusion in a block. + repeated note.NoteInclusionInBlockProof unauthenticated_note_proofs = 2; + + // The serialized chain MMR which includes proofs for all blocks referenced by the + // above note inclusion proofs as well as proofs for inclusion of the blocks referenced + // by the batches in the block. + bytes chain_mmr = 3; // The hashes of the requested accounts and their authentication paths. - repeated AccountBlockInputRecord account_states = 3; + repeated AccountWitness account_witnesses = 4; // The requested nullifiers and their authentication paths. - repeated NullifierBlockInputRecord nullifiers = 4; - - // The list of requested notes which were found in the database. - note.NoteAuthenticationInfo found_unauthenticated_notes = 5; + repeated NullifierWitness nullifier_witnesses = 5; } // Represents the result of getting batch inputs.