diff --git a/Cargo.lock b/Cargo.lock index b4b10f3c4..b728030d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ [[package]] name = "aggregator" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "ark-std", "env_logger 0.10.0", @@ -418,7 +418,7 @@ checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bus-mapping" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -1118,7 +1118,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "ethers-core 0.17.0", "ethers-signers", @@ -1384,7 +1384,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "eth-types", "geth-utils", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "digest 0.7.6", "eth-types", @@ -1653,7 +1653,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "env_logger 0.9.3", "gobuild 0.1.0-alpha.2 (git+https://github.com/scroll-tech/gobuild.git)", @@ -2243,7 +2243,7 @@ dependencies = [ [[package]] name = "keccak256" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "env_logger 0.9.3", "eth-types", @@ -2443,7 +2443,7 @@ dependencies = [ [[package]] name = "mock" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -2458,7 +2458,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "bus-mapping", "eth-types", @@ -4683,7 +4683,7 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" [[package]] name = "zkevm-circuits" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "array-init", "bus-mapping", diff --git a/ffi/src/prove.rs b/ffi/src/prove.rs index 8b62b7e46..9bf1cf7f0 100644 --- a/ffi/src/prove.rs +++ b/ffi/src/prove.rs @@ -1,21 +1,29 @@ use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char}; use libc::c_char; -use prover::{aggregator, config::ALL_AGG_DEGREES, utils::init_env_and_log, zkevm}; -use std::cell::OnceCell; +use once_cell::sync::Lazy; +use prover::{ + aggregator::{self, ChunkHash}, + config::AGG_DEGREES, + utils::init_env_and_log, + zkevm, Proof, +}; +use std::{cell::OnceCell, env}; use types::eth::BlockTrace; -static mut CHUNK_PROVER: OnceCell = OnceCell::new(); +static mut ZKEVM_PROVER: OnceCell = OnceCell::new(); static mut AGG_PROVER: OnceCell = OnceCell::new(); -static mut AGG_CHUNK_TRACES: OnceCell>> = OnceCell::new(); + +// Only used for debugging. +static OUTPUT_DIR: Lazy> = Lazy::new(|| env::var("PROVER_OUTPUT_DIR").ok()); /// # Safety #[no_mangle] -pub unsafe extern "C" fn init_chunk_prover(params_dir: *const c_char) { - init_env_and_log("ffi_chunk_prove"); +pub unsafe extern "C" fn init_zkevm_prover(params_dir: *const c_char) { + init_env_and_log("ffi_zkevm_prove"); let params_dir = c_char_to_str(params_dir); let prover = zkevm::Prover::from_params_dir(params_dir); - CHUNK_PROVER.set(prover).unwrap(); + ZKEVM_PROVER.set(prover).unwrap(); } /// # Safety @@ -25,72 +33,64 @@ pub unsafe extern "C" fn init_agg_prover(params_dir: *const c_char) { let params_dir = c_char_to_str(params_dir); - let prover = aggregator::Prover::from_params_dir(params_dir, &ALL_AGG_DEGREES); + let prover = aggregator::Prover::from_params_dir(params_dir, &AGG_DEGREES); AGG_PROVER.set(prover).unwrap(); } /// # Safety #[no_mangle] -pub unsafe extern "C" fn create_block_proof(trace_char: *const c_char) -> *const c_char { - let trace_vec = c_char_to_vec(trace_char); - let trace = serde_json::from_slice::(&trace_vec).unwrap(); - let proof = CHUNK_PROVER +pub unsafe extern "C" fn create_block_proof(block_trace: *const c_char) -> *const c_char { + let block_trace = c_char_to_vec(block_trace); + let block_trace = serde_json::from_slice::(&block_trace).unwrap(); + + let proof = ZKEVM_PROVER .get_mut() .unwrap() - .gen_chunk_proof(&[trace]) + .gen_chunk_proof(&[block_trace]) .unwrap(); + let proof_bytes = serde_json::to_vec(&proof).unwrap(); vec_to_c_char(proof_bytes) } /// # Safety #[no_mangle] -pub unsafe extern "C" fn create_chunk_proof(trace_char: *const c_char) -> *const c_char { - let trace_vec = c_char_to_vec(trace_char); - let traces = serde_json::from_slice::>(&trace_vec).unwrap(); - let proof = CHUNK_PROVER +pub unsafe extern "C" fn create_chunk_proof(block_traces: *const c_char) -> *const c_char { + let block_traces = c_char_to_vec(block_traces); + let block_traces = serde_json::from_slice::>(&block_traces).unwrap(); + + let proof = ZKEVM_PROVER .get_mut() .unwrap() - .gen_chunk_proof(traces.as_slice()) + .gen_chunk_proof(block_traces.as_slice()) .unwrap(); + let proof_bytes = serde_json::to_vec(&proof).unwrap(); vec_to_c_char(proof_bytes) } /// # Safety #[no_mangle] -pub unsafe extern "C" fn add_agg_chunk_trace(trace_char: *const c_char) { - let trace_vec = c_char_to_vec(trace_char); - let trace = serde_json::from_slice::>(&trace_vec).unwrap(); - - AGG_CHUNK_TRACES - .get_mut() - .or_else(|| { - AGG_CHUNK_TRACES.set(vec![]).unwrap(); - AGG_CHUNK_TRACES.get_mut() - }) - .unwrap() - .push(trace); -} +pub unsafe extern "C" fn create_agg_proof( + chunk_hashes: *const c_char, + chunk_proofs: *const c_char, +) -> *const c_char { + let chunk_hashes = c_char_to_vec(chunk_hashes); + let chunk_proofs = c_char_to_vec(chunk_proofs); -/// # Safety -#[no_mangle] -pub unsafe extern "C" fn clear_agg_chunk_traces() { - if let Some(chunk_traces) = AGG_CHUNK_TRACES.get_mut() { - chunk_traces.clear(); - } -} + let chunk_hashes = serde_json::from_slice::>(&chunk_hashes).unwrap(); + let chunk_proofs = serde_json::from_slice::>(&chunk_proofs).unwrap(); + assert_eq!(chunk_hashes.len(), chunk_proofs.len()); -/// # Safety -#[no_mangle] -pub unsafe extern "C" fn create_agg_proof() -> *const c_char { - // Consume the chunk traces (take and clear). - let chunk_traces = AGG_CHUNK_TRACES.take().unwrap(); + let chunks = chunk_hashes + .into_iter() + .zip(chunk_proofs.into_iter()) + .collect(); let proof = AGG_PROVER .get_mut() .unwrap() - .gen_agg_proof(chunk_traces, None) + .gen_agg_proof(chunks, OUTPUT_DIR.as_deref()) .unwrap(); let proof_bytes = serde_json::to_vec(&proof).unwrap(); diff --git a/ffi/src/verify.rs b/ffi/src/verify.rs index ab919dd8f..5b4467eb6 100644 --- a/ffi/src/verify.rs +++ b/ffi/src/verify.rs @@ -1,6 +1,6 @@ use crate::utils::{c_char_to_str, c_char_to_vec}; use libc::c_char; -use prover::{aggregator, config::AGG_LAYER4_DEGREE, utils::init_env_and_log, zkevm, Proof}; +use prover::{aggregator, config::LAYER4_DEGREE, utils::init_env_and_log, zkevm, Proof}; use std::{fs::File, io::Read}; static mut CHUNK_VERIFIER: Option<&zkevm::Verifier> = None; @@ -35,7 +35,7 @@ pub unsafe extern "C" fn init_agg_verifier(params_dir: *const c_char, vk_path: * let params_dir = c_char_to_str(params_dir); let verifier = Box::new(aggregator::Verifier::from_params_dir( params_dir, - *AGG_LAYER4_DEGREE, + *LAYER4_DEGREE, Some(vk), )); diff --git a/prover/configs/agg_layer1.config b/prover/configs/layer1.config similarity index 100% rename from prover/configs/agg_layer1.config rename to prover/configs/layer1.config diff --git a/prover/configs/agg_layer2.config b/prover/configs/layer2.config similarity index 100% rename from prover/configs/agg_layer2.config rename to prover/configs/layer2.config diff --git a/prover/configs/agg_layer3.config b/prover/configs/layer3.config similarity index 100% rename from prover/configs/agg_layer3.config rename to prover/configs/layer3.config diff --git a/prover/configs/agg_layer4.config b/prover/configs/layer4.config similarity index 100% rename from prover/configs/agg_layer4.config rename to prover/configs/layer4.config diff --git a/prover/src/aggregator.rs b/prover/src/aggregator.rs index ac0279ed5..ca8b5acb1 100644 --- a/prover/src/aggregator.rs +++ b/prover/src/aggregator.rs @@ -2,3 +2,4 @@ mod prover; mod verifier; pub use self::{prover::Prover, verifier::Verifier}; +pub use aggregator::ChunkHash; diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index d8371a5c7..81674b4cd 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -1,9 +1,9 @@ use crate::{ - config::{AGG_LAYER1_DEGREE, AGG_LAYER2_DEGREE, AGG_LAYER3_DEGREE, AGG_LAYER4_DEGREE}, - utils::{chunk_trace_to_witness_block, gen_rng, load_params, param_path_for_degree}, - zkevm::circuit::SuperCircuit, + config::{LAYER1_DEGREE, LAYER2_DEGREE, LAYER3_DEGREE, LAYER4_DEGREE}, + utils::{chunk_trace_to_witness_block, load_params, param_path_for_degree}, Proof, }; +use aggregator::{ChunkHash, MAX_AGG_SNARKS}; use anyhow::Result; use halo2_proofs::{ halo2curves::bn256::{Bn256, G1Affine}, @@ -12,14 +12,15 @@ use halo2_proofs::{ }; use std::{ collections::{BTreeMap, BTreeSet, HashMap}, - env::set_var, + iter::repeat, }; use types::eth::BlockTrace; mod aggregation; -mod chunk; mod common; mod compression; +mod evm; +mod inner; mod padding; #[derive(Debug)] @@ -78,61 +79,112 @@ impl Prover { pub fn gen_agg_proof( &mut self, - chunk_traces: Vec>, - _output_dir: Option<&str>, + chunks: Vec<(ChunkHash, Proof)>, + output_dir: Option<&str>, ) -> Result { - // Convert chunk traces to witness blocks. - let witness_blocks = chunk_traces - .into_iter() - .map(chunk_trace_to_witness_block) - .collect::>>()?; - - // Convert witness blocks to chunk hashes. - let real_chunk_hashes: Vec<_> = witness_blocks.iter().map(Into::into).collect(); - - // Generate chunk snarks. - let chunk_snarks = witness_blocks - .into_iter() - .map(|block| self.gen_chunk_snark::(&block)) - .collect::>>()?; + let real_chunk_count = chunks.len(); + assert!((1..=MAX_AGG_SNARKS).contains(&real_chunk_count)); + + let last_real_chunk_hash = chunks.last().unwrap().0; + let name = last_real_chunk_hash + .public_input_hash() + .to_low_u64_le() + .to_string(); + + // Load or generate padding snark (layer-1). + let layer1_padding_snark = + self.load_or_gen_padding_snark(&name, &last_real_chunk_hash, output_dir)?; + log::info!("Got padding snark (layer-1): {name}"); + + // Load or generate compression thin snark for padding (layer-2). + let layer2_padding_snark = self.load_or_gen_comp_snark( + &name, + "layer2", + false, + *LAYER2_DEGREE, + layer1_padding_snark, + output_dir, + )?; + log::info!("Got compression thin snark for padding (layer-2): {name}"); - // Generate compression wide snarks (layer-1). - set_var("COMPRESSION_CONFIG", "./configs/agg_layer1.config"); - let layer1_snarks = chunk_snarks - .into_iter() - .map(|snark| { - let rng = gen_rng(); - self.gen_comp_snark("agg_layer1", true, *AGG_LAYER1_DEGREE, rng, snark) - }) - .collect::>>()?; - - // Generate compression thin snarks (layer-2). - set_var("COMPRESSION_CONFIG", "./configs/agg_layer2.config"); - let layer2_snarks: Vec<_> = layer1_snarks + let (chunk_hashes, mut layer2_snarks): (Vec<_>, Vec<_>) = chunks .into_iter() - .map(|snark| { - let rng = gen_rng(); - self.gen_comp_snark("agg_layer2", false, *AGG_LAYER2_DEGREE, rng, snark) - }) - .collect::>>()?; - - // Generate aggregation snark (layer-3). - set_var("AGGREGATION_CONFIG", "./configs/agg_layer3.config"); - let rng = gen_rng(); - let layer3_snark = self.gen_agg_snark( - "agg_layer3", - *AGG_LAYER3_DEGREE, - rng, - &real_chunk_hashes, + .map(|chunk| (chunk.0, chunk.1.to_snark())) + .unzip(); + + // Extend to MAX_AGG_SNARKS by copying the padding snark. + layer2_snarks.extend(repeat(layer2_padding_snark).take(MAX_AGG_SNARKS - real_chunk_count)); + + // Load or generate aggregation snark (layer-3). + let layer3_snark = self.load_or_gen_agg_snark( + &name, + "layer3", + *LAYER3_DEGREE, + &chunk_hashes, &layer2_snarks, + output_dir, + )?; + log::info!("Got aggregation snark (layer-3): {name}"); + + // Load or generate final compression thin snark (layer-4). + let layer4_snark = self.load_or_gen_comp_snark( + &name, + "layer4", + false, + *LAYER4_DEGREE, + layer3_snark, + output_dir, )?; + log::info!("Got final compression thin snark (layer-4): {name}"); - // Generate final compression snarks (layer-4). - set_var("COMPRESSION_CONFIG", "./configs/agg_layer4.config"); - let rng = gen_rng(); - let layer4_snark = - self.gen_comp_snark("agg_layer4", false, *AGG_LAYER4_DEGREE, rng, layer3_snark)?; + let pk = self.pk("layer4").unwrap(); + Proof::from_snark(pk, &layer4_snark) + } + + pub fn gen_chunk_proof( + &mut self, + chunk_trace: Vec, + output_dir: Option<&str>, + ) -> Result { + assert!(!chunk_trace.is_empty()); + + let witness_block = chunk_trace_to_witness_block(chunk_trace)?; + log::info!("Got witness block"); + + let name = witness_block + .context + .first_or_default() + .number + .low_u64() + .to_string(); + + // Load or generate inner snark. + let inner_snark = self.load_or_gen_inner_snark(&name, witness_block, output_dir)?; + log::info!("Got inner snark: {name}"); + + // Load or generate compression wide snark (layer-1). + let layer1_snark = self.load_or_gen_comp_snark( + &name, + "layer1", + true, + *LAYER1_DEGREE, + inner_snark, + output_dir, + )?; + log::info!("Got compression wide snark (layer-1): {name}"); + + // Load or generate compression thin snark (layer-2). + let layer2_snark = self.load_or_gen_comp_snark( + &name, + "layer2", + false, + *LAYER2_DEGREE, + layer1_snark, + output_dir, + )?; + log::info!("Got compression thin snark (layer-2): {name}"); - Proof::from_snark(&self.pk_map["agg_layer4"], &layer4_snark) + let pk = self.pk("layer2").unwrap(); + Proof::from_snark(pk, &layer2_snark) } } diff --git a/prover/src/aggregator/prover/aggregation.rs b/prover/src/aggregator/prover/aggregation.rs index 4eb98630a..277d1fcac 100644 --- a/prover/src/aggregator/prover/aggregation.rs +++ b/prover/src/aggregator/prover/aggregation.rs @@ -1,8 +1,13 @@ use super::Prover; +use crate::{ + io::{load_snark, write_snark}, + utils::gen_rng, +}; use aggregator::{AggregationCircuit, BatchHash, ChunkHash}; use anyhow::Result; use rand::Rng; use snark_verifier_sdk::Snark; +use std::env::set_var; impl Prover { pub fn gen_agg_snark( @@ -24,4 +29,36 @@ impl Prover { self.gen_snark(id, degree, &mut rng, circuit) } + + pub fn load_or_gen_agg_snark( + &mut self, + name: &str, + id: &str, + degree: u32, + real_chunk_hashes: &[ChunkHash], + real_and_padding_snarks: &[Snark], + output_dir: Option<&str>, + ) -> Result { + let file_path = format!( + "{}/aggregation_snark_{}.json", + output_dir.unwrap_or_default(), + name + ); + + match output_dir.and_then(|_| load_snark(&file_path).ok().flatten()) { + Some(snark) => Ok(snark), + None => { + set_var("AGGREGATION_CONFIG", format!("./configs/{id}.config")); + + let rng = gen_rng(); + let result = + self.gen_agg_snark(id, degree, rng, real_chunk_hashes, real_and_padding_snarks); + if let (Some(_), Ok(snark)) = (output_dir, &result) { + write_snark(&file_path, snark); + } + + result + } + } + } } diff --git a/prover/src/aggregator/prover/compression.rs b/prover/src/aggregator/prover/compression.rs index 9a04df849..e1ef764af 100644 --- a/prover/src/aggregator/prover/compression.rs +++ b/prover/src/aggregator/prover/compression.rs @@ -1,9 +1,13 @@ use super::Prover; -use crate::Proof; +use crate::{ + io::{load_snark, write_snark}, + utils::gen_rng, +}; use aggregator::CompressionCircuit; use anyhow::{anyhow, Result}; use rand::Rng; use snark_verifier_sdk::Snark; +use std::env::set_var; impl Prover { pub fn gen_comp_snark( @@ -20,17 +24,34 @@ impl Prover { self.gen_snark(id, degree, &mut rng, circuit) } - pub fn gen_comp_evm_proof( + pub fn load_or_gen_comp_snark( &mut self, + name: &str, id: &str, is_fresh: bool, degree: u32, - mut rng: impl Rng + Send, prev_snark: Snark, - ) -> Result { - let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng) - .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; + output_dir: Option<&str>, + ) -> Result { + let file_path = format!( + "{}/compression_snark_{}.json", + output_dir.unwrap_or_default(), + name + ); + + match output_dir.and_then(|_| load_snark(&file_path).ok().flatten()) { + Some(snark) => Ok(snark), + None => { + set_var("COMPRESSION_CONFIG", format!("./configs/{id}.config")); + + let rng = gen_rng(); + let result = self.gen_comp_snark(id, is_fresh, degree, rng, prev_snark); + if let (Some(_), Ok(snark)) = (output_dir, &result) { + write_snark(&file_path, snark); + } - self.gen_evm_proof(id, degree, &mut rng, circuit) + result + } + } } } diff --git a/prover/src/aggregator/prover/evm.rs b/prover/src/aggregator/prover/evm.rs new file mode 100644 index 000000000..ad83d6d5f --- /dev/null +++ b/prover/src/aggregator/prover/evm.rs @@ -0,0 +1,32 @@ +use super::Prover; +use crate::{utils::gen_rng, Proof}; +use aggregator::CompressionCircuit; +use anyhow::{anyhow, Result}; +use snark_verifier_sdk::Snark; +use std::{env::set_var, path::PathBuf}; + +impl Prover { + pub fn gen_comp_evm_proof( + &mut self, + name: &str, + id: &str, + is_fresh: bool, + degree: u32, + prev_snark: Snark, + output_dir: Option<&str>, + ) -> Result { + set_var("COMPRESSION_CONFIG", format!("./configs/{id}.config")); + + let mut rng = gen_rng(); + let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng) + .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; + + let result = self.gen_evm_proof(id, degree, &mut rng, circuit); + + if let (Some(output_dir), Ok(proof)) = (output_dir, &result) { + proof.dump(&mut PathBuf::from(output_dir), name)?; + } + + result + } +} diff --git a/prover/src/aggregator/prover/chunk.rs b/prover/src/aggregator/prover/inner.rs similarity index 74% rename from prover/src/aggregator/prover/chunk.rs rename to prover/src/aggregator/prover/inner.rs index 64b910057..82ec76a8c 100644 --- a/prover/src/aggregator/prover/chunk.rs +++ b/prover/src/aggregator/prover/inner.rs @@ -11,7 +11,7 @@ use snark_verifier_sdk::{gen_snark_shplonk, Snark}; use zkevm_circuits::evm_circuit::witness::Block; impl Prover { - pub fn gen_chunk_snark( + pub fn gen_inner_snark( &mut self, witness_block: &Block, ) -> Result { @@ -28,23 +28,27 @@ impl Prover { Ok(snark) } - pub fn load_or_gen_chunk_snark( + pub fn load_or_gen_inner_snark( &mut self, - id: &str, + name: &str, witness_block: Block, output_dir: Option<&str>, ) -> Result { - let file_path = format!("{}/{}_chunk_snark.json", output_dir.unwrap_or_default(), id); + let file_path = format!( + "{}/inner_snark_{}.json", + output_dir.unwrap_or_default(), + name + ); match output_dir.and_then(|_| load_snark(&file_path).ok().flatten()) { Some(snark) => Ok(snark), None => { - let res = self.gen_chunk_snark::(&witness_block); - if let (Some(_), Ok(snark)) = (output_dir, &res) { + let result = self.gen_inner_snark::(&witness_block); + if let (Some(_), Ok(snark)) = (output_dir, &result) { write_snark(&file_path, snark); } - res + result } } } diff --git a/prover/src/aggregator/prover/padding.rs b/prover/src/aggregator/prover/padding.rs index 341ee4aa6..671b82385 100644 --- a/prover/src/aggregator/prover/padding.rs +++ b/prover/src/aggregator/prover/padding.rs @@ -1,5 +1,9 @@ use super::Prover; -use crate::{config::INNER_DEGREE, utils::gen_rng}; +use crate::{ + config::INNER_DEGREE, + io::{load_snark, write_snark}, + utils::gen_rng, +}; use aggregator::{ChunkHash, DummyChunkHashCircuit}; use anyhow::Result; use snark_verifier_sdk::{gen_snark_shplonk, Snark}; @@ -14,4 +18,29 @@ impl Prover { Ok(snark) } + + pub fn load_or_gen_padding_snark( + &mut self, + name: &str, + last_real_chunk_hash: &ChunkHash, + output_dir: Option<&str>, + ) -> Result { + let file_path = format!( + "{}/padding_snark_{}.json", + output_dir.unwrap_or_default(), + name + ); + + match output_dir.and_then(|_| load_snark(&file_path).ok().flatten()) { + Some(snark) => Ok(snark), + None => { + let result = self.gen_padding_snark(last_real_chunk_hash); + if let (Some(_), Ok(snark)) = (output_dir, &result) { + write_snark(&file_path, snark); + } + + result + } + } + } } diff --git a/prover/src/config.rs b/prover/src/config.rs index 2b26fbe7c..8cc23aa98 100644 --- a/prover/src/config.rs +++ b/prover/src/config.rs @@ -3,20 +3,23 @@ use once_cell::sync::Lazy; use std::collections::HashSet; pub static INNER_DEGREE: Lazy = Lazy::new(|| read_env_var("INNER_DEGREE", 20)); -pub static CHUNK_DEGREE: Lazy = Lazy::new(|| read_env_var("CHUNK_DEGREE", 25)); +pub static LAYER1_DEGREE: Lazy = Lazy::new(|| read_env_var("LAYER1_DEGREE", 25)); +pub static LAYER2_DEGREE: Lazy = Lazy::new(|| read_env_var("LAYER2_DEGREE", 25)); +pub static LAYER3_DEGREE: Lazy = Lazy::new(|| read_env_var("LAYER3_DEGREE", 25)); +pub static LAYER4_DEGREE: Lazy = Lazy::new(|| read_env_var("LAYER4_DEGREE", 25)); -pub static AGG_LAYER1_DEGREE: Lazy = Lazy::new(|| read_env_var("AGG_LAYER1_DEGREE", 25)); -pub static AGG_LAYER2_DEGREE: Lazy = Lazy::new(|| read_env_var("AGG_LAYER2_DEGREE", 25)); -pub static AGG_LAYER3_DEGREE: Lazy = Lazy::new(|| read_env_var("AGG_LAYER3_DEGREE", 25)); -pub static AGG_LAYER4_DEGREE: Lazy = Lazy::new(|| read_env_var("AGG_LAYER4_DEGREE", 25)); - -pub static ALL_AGG_DEGREES: Lazy> = Lazy::new(|| { +pub static ZKEVM_DEGREES: Lazy> = Lazy::new(|| { Vec::from_iter(HashSet::from([ *INNER_DEGREE, - *CHUNK_DEGREE, - *AGG_LAYER1_DEGREE, - *AGG_LAYER2_DEGREE, - *AGG_LAYER3_DEGREE, - *AGG_LAYER4_DEGREE, + *LAYER1_DEGREE, + *LAYER2_DEGREE, + ])) +}); + +pub static AGG_DEGREES: Lazy> = Lazy::new(|| { + Vec::from_iter(HashSet::from([ + *LAYER2_DEGREE, + *LAYER3_DEGREE, + *LAYER4_DEGREE, ])) }); diff --git a/prover/src/test_util.rs b/prover/src/test_util.rs index ec9c9f020..ddd8ab225 100644 --- a/prover/src/test_util.rs +++ b/prover/src/test_util.rs @@ -2,7 +2,6 @@ use crate::utils::{get_block_trace_from_file, read_env_var}; use glob::glob; use types::eth::BlockTrace; -pub mod aggregator; pub mod mock_plonk; pub const PARAMS_DIR: &str = "./test_params"; diff --git a/prover/src/test_util/aggregator.rs b/prover/src/test_util/aggregator.rs deleted file mode 100644 index d92e1f350..000000000 --- a/prover/src/test_util/aggregator.rs +++ /dev/null @@ -1,111 +0,0 @@ -use crate::{ - aggregator::Prover, - io::{load_snark, write_snark}, - utils::gen_rng, - zkevm::circuit::SuperCircuit, - Proof, -}; -use aggregator::ChunkHash; -use halo2_proofs::halo2curves::bn256::Fr; -use snark_verifier_sdk::Snark; -use std::{env::set_var, path::PathBuf}; -use zkevm_circuits::evm_circuit::witness::Block; - -pub fn gen_comp_evm_proof( - output_dir: &str, - cfg: &str, - id: &str, - is_fresh: bool, - degree: u32, - prover: &mut Prover, - prev_snark: Snark, -) -> Proof { - set_var("COMPRESSION_CONFIG", format!("./configs/{cfg}.config")); - - let rng = gen_rng(); - let proof = prover - .gen_comp_evm_proof(id, is_fresh, degree, rng, prev_snark) - .unwrap(); - proof.dump(&mut PathBuf::from(output_dir), id).unwrap(); - - proof -} - -pub fn load_or_gen_agg_snark( - output_dir: &str, - cfg: &str, - id: &str, - degree: u32, - prover: &mut Prover, - real_chunk_hashes: &[ChunkHash], - real_and_padding_snarks: &[Snark], -) -> Snark { - set_var("AGGREGATION_CONFIG", format!("./configs/{cfg}.config")); - let file_path = format!("{output_dir}/agg_snark_{id}.json"); - - load_snark(&file_path).unwrap().unwrap_or_else(|| { - let rng = gen_rng(); - let snark = prover - .gen_agg_snark(id, degree, rng, real_chunk_hashes, real_and_padding_snarks) - .unwrap(); - write_snark(&file_path, &snark); - - snark - }) -} - -pub fn load_or_gen_chunk_snark( - output_dir: &str, - id: &str, - prover: &mut Prover, - witness_block: Block, -) -> Snark { - let file_path = format!("{output_dir}/chunk_snark_{id}.json"); - - load_snark(&file_path).unwrap().unwrap_or_else(|| { - let snark = prover - .gen_chunk_snark::(&witness_block) - .unwrap(); - write_snark(&file_path, &snark); - - snark - }) -} -pub fn load_or_gen_comp_snark( - output_dir: &str, - cfg: &str, - id: &str, - is_fresh: bool, - degree: u32, - prover: &mut Prover, - prev_snark: Snark, -) -> Snark { - set_var("COMPRESSION_CONFIG", format!("./configs/{cfg}.config")); - let file_path = format!("{output_dir}/comp_snark_{id}.json"); - - load_snark(&file_path).unwrap().unwrap_or_else(|| { - let rng = gen_rng(); - let snark = prover - .gen_comp_snark(id, is_fresh, degree, rng, prev_snark) - .unwrap(); - write_snark(&file_path, &snark); - - snark - }) -} - -pub fn load_or_gen_padding_snark( - output_dir: &str, - id: &str, - prover: &mut Prover, - last_real_chunk_hash: &ChunkHash, -) -> Snark { - let file_path = format!("{output_dir}/padding_snark_{id}.json"); - - load_snark(&file_path).unwrap().unwrap_or_else(|| { - let snark = prover.gen_padding_snark(last_real_chunk_hash).unwrap(); - write_snark(&file_path, &snark); - - snark - }) -} diff --git a/prover/src/zkevm/prover.rs b/prover/src/zkevm/prover.rs index d0ac82e7a..c8c7150db 100644 --- a/prover/src/zkevm/prover.rs +++ b/prover/src/zkevm/prover.rs @@ -2,7 +2,7 @@ use super::circuit::{ block_traces_to_witness_block, check_batch_capacity, SuperCircuit, TargetCircuit, }; use crate::{ - config::{CHUNK_DEGREE, INNER_DEGREE}, + config::INNER_DEGREE, utils::{load_params, metric_of_witness_block, read_env_var, tick}, Proof, }; @@ -38,6 +38,8 @@ extern crate procfs; pub static OPT_MEM: Lazy = Lazy::new(|| read_env_var("OPT_MEM", false)); pub static MOCK_PROVE: Lazy = Lazy::new(|| read_env_var("MOCK_PROVE", false)); +const CHUNK_DEGREE: u32 = 25; + #[derive(Debug)] // This is the aggregation prover that takes in a list of traces, produces // a proof that can be verified on chain. @@ -53,13 +55,13 @@ pub struct Prover { impl Prover { pub fn from_params(inner_params: ParamsKZG, chunk_params: ParamsKZG) -> Self { assert!(inner_params.k() == *INNER_DEGREE); - assert!(chunk_params.k() == *CHUNK_DEGREE); + assert!(chunk_params.k() == CHUNK_DEGREE); // notice that `inner_k < chunk`_k which is not necessary the case in practice log::info!( "loaded parameters for degrees {} and {}", *INNER_DEGREE, - *CHUNK_DEGREE + CHUNK_DEGREE ); // this check can be skipped since the `params` is downsized? @@ -84,9 +86,9 @@ impl Prover { } pub fn from_params_dir(params_dir: &str) -> Self { - let chunk_params = load_params(params_dir, *CHUNK_DEGREE, None).unwrap(); + let chunk_params = load_params(params_dir, CHUNK_DEGREE, None).unwrap(); let inner_params = load_params(params_dir, *INNER_DEGREE, None).unwrap_or_else(|_| { - assert!(*CHUNK_DEGREE >= *INNER_DEGREE); + assert!(CHUNK_DEGREE >= *INNER_DEGREE); log::warn!( "Optimization: download params{} to params dir", *INNER_DEGREE diff --git a/prover/src/zkevm/verifier.rs b/prover/src/zkevm/verifier.rs index a49e9cc7f..dae1d5799 100644 --- a/prover/src/zkevm/verifier.rs +++ b/prover/src/zkevm/verifier.rs @@ -1,9 +1,5 @@ use super::circuit::TargetCircuit; -use crate::{ - config::{CHUNK_DEGREE, INNER_DEGREE}, - utils::load_params, - Proof, -}; +use crate::{config::INNER_DEGREE, utils::load_params, Proof}; use anyhow::{bail, Result}; use halo2_proofs::{ halo2curves::bn256::{Bn256, G1Affine}, @@ -21,6 +17,8 @@ use snark_verifier::system::halo2::transcript::evm::EvmTranscript; use snark_verifier_sdk::{verify_snark_shplonk, AggregationCircuit, Snark}; use std::{collections::HashMap, io::Cursor}; +const CHUNK_DEGREE: u32 = 25; + pub struct Verifier { inner_params: ParamsKZG, chunk_params: ParamsKZG, @@ -51,9 +49,9 @@ impl Verifier { } pub fn from_params_dir(params_dir: &str, chunk_vk: Option>) -> Self { - let chunk_params = load_params(params_dir, *CHUNK_DEGREE, None).unwrap(); + let chunk_params = load_params(params_dir, CHUNK_DEGREE, None).unwrap(); let inner_params = load_params(params_dir, *INNER_DEGREE, None).unwrap_or_else(|_| { - assert!(*CHUNK_DEGREE >= *INNER_DEGREE); + assert!(CHUNK_DEGREE >= *INNER_DEGREE); log::warn!( "Optimization: download params{} to params dir", *INNER_DEGREE diff --git a/prover/tests/aggregation_tests.rs b/prover/tests/aggregation_tests.rs index 0a7af4ddc..40acd7860 100644 --- a/prover/tests/aggregation_tests.rs +++ b/prover/tests/aggregation_tests.rs @@ -2,15 +2,9 @@ use aggregator::{CompressionCircuit, MAX_AGG_SNARKS}; use prover::{ aggregator::{Prover, Verifier}, config::{ - AGG_LAYER1_DEGREE, AGG_LAYER2_DEGREE, AGG_LAYER3_DEGREE, AGG_LAYER4_DEGREE, ALL_AGG_DEGREES, - }, - test_util::{ - aggregator::{ - gen_comp_evm_proof, load_or_gen_agg_snark, load_or_gen_chunk_snark, - load_or_gen_comp_snark, load_or_gen_padding_snark, - }, - load_block_traces_for_test, PARAMS_DIR, + AGG_DEGREES, LAYER1_DEGREE, LAYER2_DEGREE, LAYER3_DEGREE, LAYER4_DEGREE, ZKEVM_DEGREES, }, + test_util::{load_block_traces_for_test, PARAMS_DIR}, utils::{chunk_trace_to_witness_block, init_env_and_log}, }; use std::{env::set_var, iter::repeat, path::Path}; @@ -41,42 +35,49 @@ fn test_agg_prove_verify() { let real_chunk_hashes: Vec<_> = witness_blocks.iter().map(Into::into).collect(); log::info!("Got real-chunk-hashes"); - let mut prover = Prover::from_params_dir(PARAMS_DIR, &*ALL_AGG_DEGREES); - log::info!("Constructed prover"); + let mut zkevm_prover = Prover::from_params_dir(PARAMS_DIR, &*ZKEVM_DEGREES); + let mut agg_prover = Prover::from_params_dir(PARAMS_DIR, &*AGG_DEGREES); + log::info!("Constructed zkevm and aggregation provers"); - // Load or generate real chunk snarks. - let chunk_snarks: Vec<_> = witness_blocks + // Load or generate real inner snarks. + let inner_snarks: Vec<_> = witness_blocks .into_iter() .enumerate() - .map(|(i, block)| load_or_gen_chunk_snark(&output_dir, &i.to_string(), &mut prover, block)) + .map(|(i, witness_block)| { + zkevm_prover + .load_or_gen_inner_snark(&format!("layer0_{i}"), witness_block, Some(&output_dir)) + .unwrap() + }) .collect(); - log::info!("Got real-chunk-snarks"); + log::info!("Got real-inner-snarks"); // Load or generate compression wide snarks (layer-1). - let mut layer1_snarks: Vec<_> = chunk_snarks + let mut layer1_snarks: Vec<_> = inner_snarks .into_iter() .enumerate() .map(|(i, snark)| { - load_or_gen_comp_snark( - &output_dir, - "agg_layer1", - &format!("layer1_{i}"), - true, - *AGG_LAYER1_DEGREE, - &mut prover, - snark, - ) + zkevm_prover + .load_or_gen_comp_snark( + &format!("layer1_{i}"), + "layer1", + true, + *LAYER1_DEGREE, + snark, + Some(&output_dir), + ) + .unwrap() }) .collect(); - log::info!("Got compression wide snarks (layer-1)"); + log::info!("Got compression-wide-snarks (layer-1)"); // Load or generate layer-1 padding snark. - let layer1_padding_snark = load_or_gen_padding_snark( - &output_dir, - "layer1", - &mut prover, - real_chunk_hashes.last().unwrap(), - ); + let layer1_padding_snark = agg_prover + .load_or_gen_padding_snark( + "layer1", + real_chunk_hashes.last().unwrap(), + Some(&output_dir), + ) + .unwrap(); layer1_snarks.push(layer1_padding_snark); log::info!("Got layer1-padding-snark"); @@ -85,52 +86,54 @@ fn test_agg_prove_verify() { .into_iter() .enumerate() .map(|(i, snark)| { - load_or_gen_comp_snark( - &output_dir, - "agg_layer2", - &format!("layer2_{i}"), - false, - *AGG_LAYER2_DEGREE, - &mut prover, - snark, - ) + zkevm_prover + .load_or_gen_comp_snark( + &format!("layer2_{i}"), + "layer2", + false, + *LAYER2_DEGREE, + snark, + Some(&output_dir), + ) + .unwrap() }) .collect(); - log::info!("Got compression thin snarks (layer-2)"); + log::info!("Got compression-thin-snarks (layer-2)"); // Extend to MAX_AGG_SNARKS by copying the last padding snark. - let layer2_padding_snarks: Vec<_> = repeat(layer2_snarks.last().unwrap().clone()) - .take(MAX_AGG_SNARKS - layer2_snarks.len()) - .collect(); - layer2_snarks.extend(layer2_padding_snarks); + layer2_snarks.extend( + repeat(layer2_snarks.last().unwrap().clone()).take(MAX_AGG_SNARKS - layer2_snarks.len()), + ); // Load or generate aggregation snark (layer-3). - let layer3_snark = load_or_gen_agg_snark( - &output_dir, - "agg_layer3", - "layer3_0", - *AGG_LAYER3_DEGREE, - &mut prover, - &real_chunk_hashes, - &layer2_snarks, - ); - log::info!("Got aggregation snark (layer-3)"); + let layer3_snark = agg_prover + .load_or_gen_agg_snark( + "layer3_0", + "layer3", + *LAYER3_DEGREE, + &real_chunk_hashes, + &layer2_snarks, + Some(&output_dir), + ) + .unwrap(); + log::info!("Got aggregation-snark (layer-3)"); // Load or generate compression EVM proof (layer-4). - let proof = gen_comp_evm_proof( - &output_dir, - "agg_layer4", - "layer4_0", - false, - *AGG_LAYER4_DEGREE, - &mut prover, - layer3_snark, - ); - log::info!("Got compression EVM proof (layer-4)"); + let proof = agg_prover + .gen_comp_evm_proof( + "layer4_0", + "layer4", + false, + *LAYER4_DEGREE, + layer3_snark, + Some(&output_dir), + ) + .unwrap(); + log::info!("Got compression-EVM-proof (layer-4)"); // Construct verifier and EVM verify. - let params = prover.params(*AGG_LAYER4_DEGREE).clone(); - let vk = prover.pk("layer4_0").unwrap().get_vk().clone(); + let params = agg_prover.params(*LAYER4_DEGREE).clone(); + let vk = agg_prover.pk("layer4").unwrap().get_vk().clone(); let verifier = Verifier::new(params, Some(vk)); let yul_file_path = format!("{output_dir}/agg_verifier.yul"); verifier.evm_verify::(&proof, Some(Path::new(&yul_file_path))); diff --git a/prover/tests/compression_tests.rs b/prover/tests/compression_tests.rs index 3d6218db6..1a75cab97 100644 --- a/prover/tests/compression_tests.rs +++ b/prover/tests/compression_tests.rs @@ -2,16 +2,11 @@ use aggregator::CompressionCircuit; use halo2_proofs::{halo2curves::bn256::G1Affine, plonk::VerifyingKey, SerdeFormat}; use prover::{ aggregator::{Prover, Verifier}, - config::{AGG_LAYER1_DEGREE, AGG_LAYER2_DEGREE, INNER_DEGREE}, + config::{LAYER1_DEGREE, LAYER2_DEGREE, ZKEVM_DEGREES}, io::serialize_vk, - test_util::{ - aggregator::{gen_comp_evm_proof, load_or_gen_chunk_snark, load_or_gen_comp_snark}, - load_block_traces_for_test, PARAMS_DIR, - }, + test_util::{load_block_traces_for_test, PARAMS_DIR}, utils::{chunk_trace_to_witness_block, init_env_and_log}, }; -use snark_verifier::pcs::kzg::{Bdfg21, Kzg}; -use snark_verifier_sdk::{evm_verify, gen_evm_verifier, verify_snark_shplonk, CircuitExt}; use std::{io::Cursor, path::Path}; #[cfg(feature = "prove_verify")] @@ -28,42 +23,43 @@ fn test_comp_prove_verify() { let witness_block = chunk_trace_to_witness_block(chunk_trace).unwrap(); log::info!("Got witness-block"); - let mut prover = Prover::from_params_dir( - PARAMS_DIR, - &[*INNER_DEGREE, *AGG_LAYER1_DEGREE, *AGG_LAYER2_DEGREE], - ); - log::info!("Constructed prover"); + let mut zkevm_prover = Prover::from_params_dir(PARAMS_DIR, &*ZKEVM_DEGREES); + log::info!("Constructed zkevm-prover"); - // Load or generate chunk snark. - let chunk_snark = load_or_gen_chunk_snark(&output_dir, "0", &mut prover, witness_block); - log::info!("Got chunk-snark"); + // Load or generate inner snark. + let inner_snark = zkevm_prover + .load_or_gen_inner_snark("layer0", witness_block, Some(&output_dir)) + .unwrap(); + log::info!("Got inner-snark"); // Load or generate compression wide snark (layer-1). - let layer1_snark = load_or_gen_comp_snark( - &output_dir, - "agg_layer1", - "layer1_0", - true, - *AGG_LAYER1_DEGREE, - &mut prover, - chunk_snark, - ); - log::info!("Got compression wide snark (layer-1)"); + let layer1_snark = zkevm_prover + .load_or_gen_comp_snark( + "layer1_0", + "layer1", + true, + *LAYER1_DEGREE, + inner_snark, + Some(&output_dir), + ) + .unwrap(); + log::info!("Got compression-wide-snark (layer-1)"); // Load or generate compression EVM proof (layer-2). - let proof = gen_comp_evm_proof( - &output_dir, - "agg_layer2", - "layer2_0", - false, - *AGG_LAYER2_DEGREE, - &mut prover, - layer1_snark, - ); - log::info!("Got compression EVM proof (layer-2)"); + let proof = zkevm_prover + .gen_comp_evm_proof( + "layer2_0", + "layer2", + false, + *LAYER2_DEGREE, + layer1_snark, + Some(&output_dir), + ) + .unwrap(); + log::info!("Got compression-EVM-proof (layer-2)"); // Test vk deserialization. - let vk1 = prover.pk("layer2_0").unwrap().get_vk().clone(); + let vk1 = zkevm_prover.pk("layer2").unwrap().get_vk().clone(); let raw_vk1 = serialize_vk(&vk1); let mut vk2 = VerifyingKey::::read::<_, CompressionCircuit>( &mut Cursor::new(&raw_vk1), @@ -76,8 +72,7 @@ fn test_comp_prove_verify() { log::error!("test - vk2 = {:#?}", vk2); // Construct verifier and EVM verify. - let params = prover.params(*AGG_LAYER2_DEGREE).clone(); - // let vk = prover.pk("agg_layer2").unwrap().get_vk().clone(); + let params = zkevm_prover.params(*LAYER2_DEGREE).clone(); let verifier = Verifier::new(params, Some(vk2)); let yul_file_path = format!("{output_dir}/comp_verifier.yul"); verifier.evm_verify::(&proof, Some(Path::new(&yul_file_path)));