fix: run circuit-capacity-checker for each chunk in mock-testnet

bin/Cargo.toml

@@ -20,6 +20,7 @@ serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0.66"
 tokio = { version = "1", features = ["full"] }
+integration = { path = "../integration" }
 prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
 
 [[bin]]

bin/src/mock_testnet.rs

@@ -1,6 +1,6 @@
-#![allow(dead_code)]
 use anyhow::Result;
 use ethers_providers::{Http, Provider};
+use integration::test_util::{prepare_circuit_capacity_checker, run_circuit_capacity_checker};
 use prover::{
     inner::Prover,
     utils::init_env_and_log,
@@ -25,6 +25,9 @@ async fn main() {
     let setting = Setting::new();
     log::info!("mock-testnet: {setting:?}");
 
+    prepare_circuit_capacity_checker();
+    log::info!("mock-testnet: prepared ccc");
+
     let provider = Provider::<Http>::try_from(&setting.l2geth_api_url)
         .expect("mock-testnet: failed to initialize ethers Provider");
 
@@ -94,9 +97,12 @@ async fn main() {
 
 fn build_block(
     block_traces: &[BlockTrace],
-    _batch_id: i64,
+    batch_id: i64,
     chunk_id: i64,
 ) -> anyhow::Result<WitnessBlock> {
+    run_circuit_capacity_checker(block_traces);
+    log::info!("mock-testnet: run ccc for batch-{batch_id} chunk-{chunk_id}");
+
     let gas_total: u64 = block_traces
         .iter()
         .map(|b| b.header.gas_used.as_u64())
@@ -149,6 +155,7 @@ struct RollupscanResponse {
     chunks: Option<Vec<ChunkInfo>>,
 }
 
+#[allow(dead_code)]
 #[derive(Deserialize, Debug)]
 struct ChunkInfo {
     index: i64,

integration/Cargo.toml

@@ -6,18 +6,18 @@ edition = "2021"
 [dependencies]
 halo2_proofs = { git = "https://github.com/privacy-scaling-explorations/halo2.git", tag = "v2023_02_02" }
 prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
+zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
 
 anyhow = "1.0"
 glob = "0.3.0"
+itertools = "0.10.5"
 log = "0.4"
 log4rs = { version = "1.2.0", default_features = false, features = ["console_appender", "file_appender"] }
 rand = "0.8"
 
 [dev-dependencies]
-itertools = "0.10.5"
 serde = "1.0"
 serde_derive = "1.0"
-zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
 
 [features]
 default = []

integration/src/test_util.rs

@@ -4,9 +4,11 @@ use prover::{
     BlockTrace,
 };
 
+mod capacity_checker;
 pub mod mock_plonk;
 mod proof;
 
+pub use capacity_checker::{prepare_circuit_capacity_checker, run_circuit_capacity_checker};
 pub use proof::{
     gen_and_verify_batch_proofs, gen_and_verify_chunk_proofs, gen_and_verify_normal_and_evm_proofs,
     gen_and_verify_normal_proof,

integration/src/test_util/capacity_checker.rs

@@ -0,0 +1,124 @@
+use itertools::Itertools;
+use prover::{
+    zkevm::{CircuitCapacityChecker, RowUsage},
+    BlockTrace,
+};
+use zkevm_circuits::evm_circuit::ExecutionState;
+
+pub fn prepare_circuit_capacity_checker() {
+    // Force evm_circuit::param::EXECUTION_STATE_HEIGHT_MAP to be initialized.
+    let mulmod_height = ExecutionState::MULMOD.get_step_height();
+    log::debug!("mulmod_height {mulmod_height}");
+    debug_assert_eq!(mulmod_height, 18);
+}
+
+pub fn run_circuit_capacity_checker(blocks: &[BlockTrace]) {
+    let each_tx_ccc_result = get_ccc_result_by_each_tx(blocks);
+
+    for light_mode in [true, false] {
+        let whole_block_ccc_result = get_ccc_result_by_whole_block(light_mode, blocks);
+        check_each_tx_and_whole_block_ccc_results(&each_tx_ccc_result, &whole_block_ccc_result);
+    }
+}
+
+fn get_ccc_result_by_each_tx(blocks: &[BlockTrace]) -> RowUsage {
+    log::info!(
+        "estimating circuit rows tx by tx, tx num {}",
+        blocks
+            .iter()
+            .map(|b| b.execution_results.len())
+            .sum::<usize>(),
+    );
+
+    let mut checker = CircuitCapacityChecker::new();
+    // checker.light_mode = false;
+
+    let start_time = std::time::Instant::now();
+
+    let mut tx_num = 0;
+    for (block_idx, block) in blocks.iter().enumerate() {
+        for tx_idx in 0..block.transactions.len() {
+            log::info!("processing {}th block {}th tx", block_idx, tx_idx);
+            #[rustfmt::skip]
+            /*  
+            The capacity_checker is expected to be run inside sequencer, where we don't have the traces of blocks, instead we only have traces of tx.
+            For the "tx_trace":
+                transactions: 
+                    the tx itself. For compatibility reasons, transactions is a vector of len 1 now.   
+                execution_results: 
+                    tx execution trace. Similar with above, it is also of len 1 vevtor.   
+                storage_trace: 
+                    storage_trace is prestate + siblings(or proofs) of touched storage_slots and accounts of this tx.
+            */
+
+            let tx_trace = BlockTrace {
+                transactions: vec![block.transactions[tx_idx].clone()],
+                execution_results: vec![block.execution_results[tx_idx].clone()],
+                storage_trace: block.tx_storage_trace[tx_idx].clone(),
+                chain_id: block.chain_id,
+                coinbase: block.coinbase.clone(),
+                header: block.header.clone(),
+                start_l1_queue_index: block.start_l1_queue_index,
+                tx_storage_trace: vec![], // not used
+            };
+            log::debug!("calling estimate_circuit_capacity");
+            let results = checker.estimate_circuit_capacity(&[tx_trace]);
+            log::info!(
+                "after {}th block {}th tx: {:#?}",
+                block_idx,
+                tx_idx,
+                results
+            );
+        }
+        tx_num += block.transactions.len();
+    }
+    log::info!("capacity_checker test done");
+    let ccc_result = checker.get_acc_row_usage(false);
+    log::info!(
+        "ccc result tx by tx {:#?}, after normalize {:#?}",
+        ccc_result,
+        ccc_result.normalize()
+    );
+
+    let avg_ccc_time = start_time.elapsed().as_millis() as usize / tx_num;
+    log::info!("avg time each tx: {avg_ccc_time}ms",);
+    assert!(avg_ccc_time < 100);
+
+    ccc_result
+}
+
+fn get_ccc_result_by_whole_block(light_mode: bool, blocks: &[BlockTrace]) -> RowUsage {
+    log::info!("estimating circuit rows whole block, light_mode {light_mode}");
+    let mut checker = CircuitCapacityChecker::new();
+    checker.light_mode = light_mode;
+
+    checker.estimate_circuit_capacity(blocks).unwrap();
+    let ccc_result = checker.get_acc_row_usage(false);
+    log::info!(
+        "ccc result whole block {:#?}, after normalize {:#?}",
+        ccc_result,
+        ccc_result.normalize()
+    );
+
+    ccc_result
+}
+
+fn check_each_tx_and_whole_block_ccc_results(
+    each_tx_ccc_result: &RowUsage,
+    whole_block_ccc_result: &RowUsage,
+) {
+    for (t, b) in each_tx_ccc_result
+        .row_usage_details
+        .iter()
+        .zip_eq(whole_block_ccc_result.row_usage_details.iter())
+    {
+        log::info!(
+            "{}: {}(tx) vs {}(block), over estimate ratio {}",
+            t.name,
+            t.row_number,
+            b.row_number,
+            t.row_number as f64 / b.row_number as f64
+        );
+        assert!(t.row_number >= b.row_number);
+    }
+}

integration/tests/integration.rs

@@ -2,18 +2,16 @@ use halo2_proofs::{
     plonk::{keygen_pk2, keygen_vk},
     poly::commitment::Params,
 };
-use integration::test_util::{load_block_traces_for_test, parse_trace_path_from_mode, PARAMS_DIR};
-use itertools::Itertools;
+use integration::test_util::{
+    load_block_traces_for_test, parse_trace_path_from_mode, prepare_circuit_capacity_checker,
+    run_circuit_capacity_checker, PARAMS_DIR,
+};
 use prover::{
     config::INNER_DEGREE,
     inner::{Prover, Verifier},
     io::serialize_vk,
     utils::{get_block_trace_from_file, init_env_and_log, load_params, short_git_version},
-    zkevm::{
-        circuit::{SuperCircuit, TargetCircuit},
-        CircuitCapacityChecker,
-    },
-    BlockTrace,
+    zkevm::circuit::{SuperCircuit, TargetCircuit},
 };
 use zkevm_circuits::util::SubCircuit;
 
@@ -70,90 +68,11 @@ fn test_cs_same_for_vk_consistent() {
 fn test_capacity_checker() {
     init_env_and_log("integration");
     let trace_path = parse_trace_path_from_mode("multiswap");
-    //let trace_path = "./tests/extra_traces/new.json";
-    let batch = vec![get_block_trace_from_file(trace_path)];
-
-    // Force the evm_circuit::param::EXECUTION_STATE_HEIGHT_MAP be inited;
-    let mulmod_height = zkevm_circuits::evm_circuit::ExecutionState::MULMOD.get_step_height();
-    log::debug!("mulmod_height {mulmod_height}");
-    debug_assert_eq!(mulmod_height, 18);
+    // let trace_path = "./tests/extra_traces/new.json";
+    let blocks = vec![get_block_trace_from_file(trace_path)];
 
-    log::info!(
-        "estimating circuit rows tx by tx, tx num {}",
-        batch[0].execution_results.len()
-    );
-    let mut checker = CircuitCapacityChecker::new();
-    //checker.light_mode = false;
-    let start_time = std::time::Instant::now();
-    let mut tx_num = 0;
-    for (block_idx, block) in batch.iter().enumerate() {
-        for i in 0..block.transactions.len() {
-            log::info!("processing {}th block {}th tx", block_idx, i);
-            #[rustfmt::skip]
-            /*  
-            The capacity_checker is expected to be run inside sequencer, where we don't have the traces of blocks, instead we only have traces of tx.
-            For the "tx_trace":
-                transactions: 
-                    the tx itself. For compatibility reasons, transactions is a vector of len 1 now.   
-                execution_results: 
-                    tx execution trace. Similar with above, it is also of len 1 vevtor.   
-                storage_trace: 
-                    storage_trace is prestate + siblings(or proofs) of touched storage_slots and accounts of this tx.
-            */
-            let tx_trace = BlockTrace {
-                transactions: vec![block.transactions[i].clone()],
-                execution_results: vec![block.execution_results[i].clone()],
-                storage_trace: block.tx_storage_trace[i].clone(),
-                chain_id: block.chain_id,
-                coinbase: block.coinbase.clone(),
-                header: block.header.clone(),
-                start_l1_queue_index: block.start_l1_queue_index,
-                tx_storage_trace: Vec::new(), // not used
-            };
-            log::debug!("calling estimate_circuit_capacity");
-            let results = checker.estimate_circuit_capacity(&[tx_trace]);
-            log::info!("after {}th block {}th tx: {:#?}", block_idx, i, results);
-        }
-        tx_num += block.transactions.len();
-    }
-    log::info!("capacity_checker test done");
-    let ccc_result_tx_by_tx = checker.get_acc_row_usage(false);
-    log::info!(
-        "ccc result tx by tx {:#?}, after normalize {:#?}",
-        ccc_result_tx_by_tx,
-        ccc_result_tx_by_tx.normalize()
-    );
-    let avg_ccc_time = start_time.elapsed().as_millis() as usize / tx_num;
-    log::info!("avg time each tx: {avg_ccc_time}ms",);
-    assert!(avg_ccc_time < 100);
-
-    for light_mode in [true, false] {
-        log::info!("estimating circuit rows whole block, light_mode {light_mode}");
-        let mut checker = CircuitCapacityChecker::new();
-        checker.light_mode = light_mode;
-        checker.estimate_circuit_capacity(&batch).unwrap();
-        let ccc_result_whole_block = checker.get_acc_row_usage(false);
-        log::info!(
-            "ccc result whole block {:#?}, after normalize {:#?}",
-            ccc_result_whole_block,
-            ccc_result_whole_block.normalize()
-        );
-
-        for (t, b) in ccc_result_tx_by_tx
-            .row_usage_details
-            .iter()
-            .zip_eq(ccc_result_whole_block.row_usage_details.iter())
-        {
-            log::info!(
-                "{}: {}(tx) vs {}(block), over estimate ratio {}",
-                t.name,
-                t.row_number,
-                b.row_number,
-                t.row_number as f64 / b.row_number as f64
-            );
-            assert!(t.row_number >= b.row_number);
-        }
-    }
+    prepare_circuit_capacity_checker();
+    run_circuit_capacity_checker(&blocks);
 }
 
 #[test]