wip

Author: lispc

Cargo.lock

@@ -27,8 +27,8 @@ serde_json = "1.0"
 tokio = { version = "1.32", features = ["full"] }
 
 halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "v1.1"  }
-prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.10.0rc3", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
-zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.10.0rc3", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
+prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.10.3", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
+zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.10.3", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
 
 integration = { path = "integration" }
 

Cargo.toml

@@ -2,6 +2,8 @@ CURRENTDATE=`date +"%Y-%m-%d"`
 
 CHAIN_ID ?= 534352
 export CHAIN_ID
+RUST_MIN_STACK ?= 100000000
+export RUST_MIN_STACK
 
 help: ## Display this help screen
 	@grep -h \
@@ -44,8 +46,8 @@ test-inner-prove:
 test-chunk-prove:
 	@cargo test --features prove_verify --release test_chunk_prove_verify
 
-test-agg-prove:
-	@cargo test --features prove_verify --release test_agg_prove_verify
+test-e2e-prove:
+	@cargo test --features prove_verify --release test_e2e_prove_verify
 
 test-batch-prove:
 	@cargo test --features prove_verify --release test_batch_prove_verify

Makefile

@@ -20,51 +20,28 @@ pub use proof::{
 pub const ASSETS_DIR: &str = "./test_assets";
 pub const PARAMS_DIR: &str = "./test_params";
 
-pub fn parse_trace_path_from_mode(mode: &str) -> &'static str {
-    let trace_path = match mode {
-        "empty" => "./tests/traces/bridge/01.json",
-        "greeter" => "./tests/traces/greeter/setValue.json",
-        "single" => "./tests/traces/erc20/1_transfer.json",
-        "multiple" => "./tests/extra_traces/batch_495/chunk_495/block_8802.json",
-        "multiswap" => "./tests/traces/multi_uniswapv2/router-swapExactTokensForTokens_34.json",
-        "native" => "./tests/traces/native/transfer.json",
-        "dao" => "./tests/traces/dao/dao-propose.json",
-        "nft" => "./tests/traces/nft/mint.json",
-        "sushi" => "./tests/traces/sushi/chef-withdraw.json",
-        _ => "./tests/extra_traces/batch_495/chunk_495/block_8802.json",
-    };
-    log::info!("using mode {:?}, testing with {:?}", mode, trace_path);
-    trace_path
-}
-
 pub fn load_block_traces_for_test() -> (Vec<String>, Vec<BlockTrace>) {
-    let trace_path: String = read_env_var("TRACE_PATH", "".to_string());
-    let paths: Vec<String> = if trace_path.is_empty() {
-        // use mode
-        let mode = read_env_var("MODE", "default".to_string());
-        if mode.to_lowercase() == "batch" || mode.to_lowercase() == "pack" {
-            (1..=20)
-                .map(|i| format!("tests/traces/bridge/{i:02}.json"))
-                .collect()
-        } else {
-            vec![parse_trace_path_from_mode(&mode).to_string()]
-        }
-    } else if !std::fs::metadata(&trace_path).unwrap().is_dir() {
+    let trace_path: String = read_env_var(
+        "TRACE_PATH",
+        "./tests/extra_traces/batch_495/chunk_495/block_8802.json".to_string(),
+    );
+    let paths: Vec<String> = if !std::fs::metadata(&trace_path).unwrap().is_dir() {
         vec![trace_path]
     } else {
-        load_batch_traces(&trace_path).0
+        load_chunk_traces(&trace_path).0
     };
     log::info!("test cases traces: {:?}", paths);
     let traces: Vec<_> = paths.iter().map(get_block_trace_from_file).collect();
     (paths, traces)
 }
 
-fn load_batch_traces(batch_dir: &str) -> (Vec<String>, Vec<BlockTrace>) {
-    let file_names: Vec<String> = glob(&format!("{batch_dir}/**/*.json"))
+fn load_chunk_traces(chunk_dir: &str) -> (Vec<String>, Vec<BlockTrace>) {
+    // Nested dirs are not allowed
+    let file_names: Vec<String> = glob(&format!("{chunk_dir}/*.json"))
         .unwrap()
         .map(|p| p.unwrap().to_str().unwrap().to_string())
         .collect();
-    log::info!("test batch with {:?}", file_names);
+    log::info!("test chunk with {:?}", file_names);
     let mut names_and_traces = file_names
         .into_iter()
         .map(|trace_path| {
@@ -86,3 +63,16 @@ fn load_batch_traces(batch_dir: &str) -> (Vec<String>, Vec<BlockTrace>) {
     );
     names_and_traces.into_iter().map(|(f, t, _)| (f, t)).unzip()
 }
+
+pub fn load_batch() -> anyhow::Result<Vec<String>> {
+    let batch_dir = read_env_var("TRACE_PATH", "./tests/extra_traces/batch_24".to_string());
+    let mut sorted_dirs: Vec<String> = std::fs::read_dir(batch_dir)?
+        .filter_map(|entry| entry.ok())
+        .map(|entry| entry.path())
+        .filter(|path| path.is_dir())
+        .map(|path| path.to_string_lossy().into_owned())
+        .collect::<Vec<String>>();
+    sorted_dirs.sort();
+    log::info!("batch content: {:?}", sorted_dirs);
+    Ok(sorted_dirs)
+}

integration/src/test_util.rs

@@ -7,29 +7,34 @@ use prover::{
 };
 use std::env;
 
-pub fn gen_and_verify_batch_proofs(agg_prover: &mut Prover, layer3_snark: Snark, output_dir: &str) {
-    let evm_proof = gen_and_verify_normal_and_evm_proofs(
+pub fn gen_and_verify_batch_proofs(
+    agg_prover: &mut Prover,
+    layer3_snark: Snark,
+    output_dir: &str,
+    assets_dir: &str,
+) {
+    let (_normal_proof, evm_proof) = gen_and_verify_normal_and_evm_proofs(
         &mut agg_prover.inner,
         LayerId::Layer4,
         layer3_snark,
         Some(output_dir),
-    )
-    .1;
-    verify_batch_proof(evm_proof, output_dir);
+    );
+    verify_batch_proof(evm_proof, output_dir, assets_dir);
 }
 
+// So here even for chunk, we generate and verify a evm proof
+// TODO: remove this?
 pub fn gen_and_verify_chunk_proofs(
     zkevm_prover: &mut zkevm::Prover,
     layer1_snark: Snark,
     output_dir: &str,
 ) {
-    let normal_proof = gen_and_verify_normal_and_evm_proofs(
+    let (normal_proof, _evm_proof) = gen_and_verify_normal_and_evm_proofs(
         &mut zkevm_prover.inner,
         LayerId::Layer2,
         layer1_snark,
         Some(output_dir),
-    )
-    .0;
+    );
     verify_chunk_proof(&zkevm_prover.inner, normal_proof, output_dir);
 }
 
@@ -99,12 +104,12 @@ fn gen_normal_proof(
     snark
 }
 
-fn verify_batch_proof(evm_proof: EvmProof, output_dir: &str) {
+fn verify_batch_proof(evm_proof: EvmProof, output_dir: &str, assets_dir: &str) {
     let batch_proof = BatchProof::from(evm_proof.proof);
     batch_proof.dump(output_dir, "agg").unwrap();
     batch_proof.clone().assert_calldata();
 
-    let verifier = Verifier::from_dirs(PARAMS_DIR, output_dir);
+    let verifier = Verifier::from_dirs(PARAMS_DIR, assets_dir);
     log::info!("Constructed aggregator verifier");
 
     assert!(verifier.verify_agg_evm_proof(batch_proof));

integration/src/test_util/proof.rs

@@ -1,55 +1,100 @@
-use integration::test_util::{gen_and_verify_batch_proofs, PARAMS_DIR};
+use integration::test_util::{
+    gen_and_verify_batch_proofs, load_batch, load_block_traces_for_test, PARAMS_DIR,
+};
 use prover::{
-    aggregator::Prover, proof::from_json_file, utils::init_env_and_log, ChunkHash, ChunkProof,
+    aggregator::Prover,
+    config::LayerId,
+    proof::from_json_file,
+    utils::{chunk_trace_to_witness_block, init_env_and_log},
+    BatchHash, BatchProof, ChunkHash, ChunkProof, CompressionCircuit,
 };
 use serde_derive::{Deserialize, Serialize};
 use std::{env, fs, path::PathBuf};
 
-#[cfg(feature = "prove_verify")]
+//#[cfg(feature = "prove_verify")]
 #[test]
 fn test_batch_prove_verify() {
     let output_dir = init_env_and_log("batch_tests");
     log::info!("Initialized ENV and created output-dir {output_dir}");
 
-    let chunk_hashes_proofs = load_chunk_hashes_and_proofs("tests/test_data", "1", &output_dir);
-    let mut batch_prover = new_batch_prover(&output_dir);
-    prove_and_verify_batch(&output_dir, &mut batch_prover, chunk_hashes_proofs);
-}
+    // let batch_task = "tests/test_data/full_proof_1.json";
+    let batch_task = "tests/test_data/batch_task_69776.json";
+    let assets_dir = "assets";
+    let chunk_hashes_proofs = load_batch_proving_task(batch_task, &output_dir);
 
-#[cfg(feature = "prove_verify")]
-#[test]
-fn test_batches_with_each_chunk_num_prove_verify() {
-    let output_dir = init_env_and_log("batches_with_each_chunk_num_tests");
-    log::info!("Initialized ENV and created output-dir {output_dir}");
+    env::set_var("AGG_VK_FILENAME", "agg_vk.vkey");
+    env::set_var("CHUNK_PROTOCOL_FILENAME", "chunk.protocol");
+    let mut batch_prover = Prover::from_dirs(PARAMS_DIR, assets_dir);
+    log::info!("Constructed batch prover");
 
-    let chunk_hashes_proofs = load_chunk_hashes_and_proofs("tests/test_data", "2", &output_dir);
-    let mut batch_prover = new_batch_prover(&output_dir);
+    let chunk_num = chunk_hashes_proofs.len();
+    log::info!("Prove batch BEGIN: chunk_num = {chunk_num}");
 
-    // Iterate over chunk proofs to test with 1 - 15 chunks (in a batch).
-    for i in 0..chunk_hashes_proofs.len() {
-        let mut output_dir = PathBuf::from(&output_dir);
-        output_dir.push(format!("batch_{}", i + 1));
-        fs::create_dir_all(&output_dir).unwrap();
+    // Load or generate aggregation snark (layer-3).
+    let layer3_snark = batch_prover
+        .load_or_gen_last_agg_snark("agg", chunk_hashes_proofs, Some(&output_dir))
+        .unwrap();
 
-        prove_and_verify_batch(
-            &output_dir.to_string_lossy(),
-            &mut batch_prover,
-            chunk_hashes_proofs[..=i].to_vec(),
-        );
-    }
+    let layer_id = LayerId::Layer4;
+
+    let id = layer_id.id();
+    let degree = layer_id.degree();
+    // Load or generate compression snark.
+    let normal_proof = batch_prover
+        .inner
+        .load_or_gen_comp_snark(
+            "normal",
+            id,
+            true,
+            degree,
+            layer3_snark.clone(),
+            Some(&output_dir),
+        )
+        .unwrap();
+    log::info!("Generated compression snark: {id}");
+
+    let evm_proof = batch_prover
+        .inner
+        .load_or_gen_comp_evm_proof("evm", id, true, degree, layer3_snark, Some(&output_dir))
+        .unwrap();
+    log::info!("Generated EVM proof: {id}");
+
+    let config_path = layer_id.config_path();
+
+    env::set_var("COMPRESSION_CONFIG", config_path);
+    let vk = evm_proof.proof.vk::<CompressionCircuit>();
+
+    let params = batch_prover.inner.params(degree).clone();
+    let verifier = prover::common::Verifier::<CompressionCircuit>::new(params, vk);
+    log::info!("Constructed common verifier");
+
+    assert!(verifier.verify_snark(normal_proof));
+    log::info!("Verified normal proof: {id}");
+
+    verifier.evm_verify(&evm_proof, Some(&output_dir));
+    log::info!("Verified EVM proof: {id}");
+
+    let batch_proof = BatchProof::from(evm_proof.proof);
+    batch_proof.dump(&output_dir, "agg").unwrap();
+    batch_proof.clone().assert_calldata();
+
+    let verifier = prover::aggregator::Verifier::from_dirs(PARAMS_DIR, assets_dir);
+    log::info!("Constructed aggregator verifier");
+
+    assert!(verifier.verify_agg_evm_proof(batch_proof));
+    log::info!("Verified batch proof");
+
+    log::info!("Prove batch END: chunk_num = {chunk_num}");
 }
+
 #[derive(Debug, Deserialize, Serialize)]
 struct BatchTaskDetail {
     chunk_infos: Vec<ChunkHash>,
     chunk_proofs: Vec<ChunkProof>,
 }
 
-fn load_chunk_hashes_and_proofs(
-    dir: &str,
-    filename: &str,
-    output_dir: &str,
-) -> Vec<(ChunkHash, ChunkProof)> {
-    let batch_task_detail: BatchTaskDetail = from_json_file(dir, filename).unwrap();
+fn load_batch_proving_task(filename: &str, output_dir: &str) -> Vec<(ChunkHash, ChunkProof)> {
+    let batch_task_detail: BatchTaskDetail = from_json_file(filename).unwrap();
     let chunk_hashes = batch_task_detail.chunk_infos;
     let chunk_proofs = batch_task_detail.chunk_proofs;
 
@@ -59,13 +104,13 @@ fn load_chunk_hashes_and_proofs(
         .zip(chunk_proofs[..].iter().cloned())
         .collect();
 
-    // Dump chunk-procotol for further batch-proving.
-    chunk_hashes_proofs
-        .first()
-        .unwrap()
-        .1
-        .dump(output_dir, "0")
-        .unwrap();
+    let dump_protocol = false;
+    if dump_protocol {
+        log::info!("dumping first chunk protocol to {output_dir}/chunk_protocol");
+        let chunk_protocol = &chunk_hashes_proofs.first().unwrap().1.protocol;
+        // Dump chunk-procotol for further batch-proving.
+        prover::proof::dump_data(output_dir, "chunk.protocol", &chunk_protocol);
+    }
 
     log::info!(
         "Loaded chunk-hashes and chunk-proofs: total = {}",
@@ -74,29 +119,48 @@ fn load_chunk_hashes_and_proofs(
     chunk_hashes_proofs
 }
 
-fn new_batch_prover(assets_dir: &str) -> Prover {
-    env::set_var("AGG_VK_FILENAME", "vk_batch_agg.vkey");
-    env::set_var("CHUNK_PROTOCOL_FILENAME", "chunk_chunk_0.protocol");
-    let prover = Prover::from_dirs(PARAMS_DIR, assets_dir);
-    log::info!("Constructed batch prover");
+#[test]
+fn test_batch_pi_consistency() {
+    let output_dir = init_env_and_log("batch_pi");
+    log::info!("Initialized ENV and created output-dir {output_dir}");
+    let trace_paths = load_batch().unwrap();
 
-    prover
-}
+    let max_num_snarks = 15;
+    let chunk_traces: Vec<_> = trace_paths
+        .iter()
+        .map(|trace_path| {
+            env::set_var("TRACE_PATH", trace_path);
+            load_block_traces_for_test().1
+        })
+        .collect();
 
-fn prove_and_verify_batch(
-    output_dir: &str,
-    batch_prover: &mut Prover,
-    chunk_hashes_proofs: Vec<(ChunkHash, ChunkProof)>,
-) {
-    let chunk_num = chunk_hashes_proofs.len();
-    log::info!("Prove batch BEGIN: chunk_num = {chunk_num}");
+    let mut chunk_hashes: Vec<ChunkHash> = chunk_traces
+        .into_iter()
+        .enumerate()
+        .map(|(_i, chunk_trace)| {
+            let witness_block = chunk_trace_to_witness_block(chunk_trace.clone()).unwrap();
+            ChunkHash::from_witness_block(&witness_block, false)
+        })
+        .collect();
 
-    // Load or generate aggregation snark (layer-3).
-    let layer3_snark = batch_prover
-        .load_or_gen_last_agg_snark("agg", chunk_hashes_proofs, Some(output_dir))
-        .unwrap();
+    let real_chunk_count = chunk_hashes.len();
+    if real_chunk_count < max_num_snarks {
+        let mut padding_chunk_hash = chunk_hashes.last().unwrap().clone();
+        padding_chunk_hash.is_padding = true;
 
-    gen_and_verify_batch_proofs(batch_prover, layer3_snark, output_dir);
+        // Extend to MAX_AGG_SNARKS for both chunk hashes and layer-2 snarks.
+        chunk_hashes
+            .extend(std::iter::repeat(padding_chunk_hash).take(max_num_snarks - real_chunk_count));
+    }
 
-    log::info!("Prove batch END: chunk_num = {chunk_num}");
+    let batch_hash = BatchHash::construct(&chunk_hashes);
+    let blob = batch_hash.blob_assignments();
+
+    let challenge = blob.challenge;
+    let evaluation = blob.evaluation;
+    println!("blob.challenge: {challenge:x}");
+    println!("blob.evaluation: {evaluation:x}");
+    for (i, elem) in blob.coefficients.iter().enumerate() {
+        println!("blob.coeffs[{}]: {elem:x}", i);
+    }
 }

integration/tests/aggregation_tests.rs

@@ -0,0 +1,78 @@
+use integration::test_util::{
+    gen_and_verify_batch_proofs, load_batch, load_block_traces_for_test, ASSETS_DIR, PARAMS_DIR,
+};
+use prover::{
+    aggregator::Prover,
+    utils::{chunk_trace_to_witness_block, init_env_and_log, read_env_var},
+    zkevm, BatchHash, BlockTrace, ChunkHash, ChunkProof,
+};
+use std::env;
+
+#[cfg(feature = "prove_verify")]
+#[test]
+fn test_e2e_prove_verify() {
+    let output_dir = init_env_and_log("e2e_tests");
+    log::info!("Initialized ENV and created output-dir {output_dir}");
+
+    let chunk_paths = load_batch().unwrap();
+    let chunk_hashes_proofs = gen_chunk_hashes_and_proofs(&output_dir, &chunk_paths);
+
+    let mut batch_prover = new_batch_prover(&output_dir);
+    prove_and_verify_batch(&output_dir, &mut batch_prover, chunk_hashes_proofs);
+}
+
+fn gen_chunk_hashes_and_proofs(
+    output_dir: &str,
+    chunk_paths: &[String],
+) -> Vec<(ChunkHash, ChunkProof)> {
+    let mut zkevm_prover = zkevm::Prover::from_dirs(PARAMS_DIR, ASSETS_DIR);
+    log::info!("Constructed zkevm prover");
+
+    let chunk_traces: Vec<Vec<BlockTrace>> = chunk_paths
+        .iter()
+        .map(|chunk_path| {
+            env::set_var("TRACE_PATH", chunk_path);
+            load_block_traces_for_test().1
+        })
+        .collect();
+
+    let chunk_hashes_proofs = chunk_traces
+        .into_iter()
+        .enumerate()
+        .map(|(i, chunk_trace)| {
+            let witness_block = chunk_trace_to_witness_block(chunk_trace.clone()).unwrap();
+            let chunk_hash = ChunkHash::from_witness_block(&witness_block, false);
+
+            let proof = zkevm_prover
+                .gen_chunk_proof(chunk_trace, Some(&i.to_string()), None, Some(output_dir))
+                .unwrap();
+
+            (chunk_hash, proof)
+        })
+        .collect();
+
+    log::info!("Generated chunk hashes and proofs");
+    chunk_hashes_proofs
+}
+
+fn new_batch_prover(assets_dir: &str) -> Prover {
+    env::set_var("AGG_VK_FILENAME", "vk_batch_agg.vkey");
+    env::set_var("CHUNK_PROTOCOL_FILENAME", "chunk_chunk_0.protocol");
+    let prover = Prover::from_dirs(PARAMS_DIR, assets_dir);
+    log::info!("Constructed batch prover");
+
+    prover
+}
+
+fn prove_and_verify_batch(
+    output_dir: &str,
+    batch_prover: &mut Prover,
+    chunk_hashes_proofs: Vec<(ChunkHash, ChunkProof)>,
+) {
+    // Load or generate aggregation snark (layer-3).
+    let layer3_snark = batch_prover
+        .load_or_gen_last_agg_snark("agg", chunk_hashes_proofs, Some(output_dir))
+        .unwrap();
+
+    gen_and_verify_batch_proofs(batch_prover, layer3_snark, output_dir, output_dir);
+}