Revert "Revert "wip; refactor and l1 fee""

This reverts commit 641ac1c.

Cargo.toml

@@ -27,8 +27,8 @@ serde_json = "1.0"
 tokio = { version = "1.32", features = ["full"] }
 
 halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "v1.1"  }
-prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.11.0rc4", default-features = false, features = ["parallel_syn", "scroll"] }
-zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.11.0rc4", default-features = false, features = ["parallel_syn", "scroll"] }
+prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "feat/refactor-prover", default-features = false, features = ["parallel_syn", "scroll"] }
+zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch ="feat/refactor-prover", default-features = false, features = ["parallel_syn", "scroll", "l1_fee_curie"] }
 
 integration = { path = "integration" }
 

Makefile

@@ -40,7 +40,7 @@ test-chunk-prove:
 	@cargo test --features prove_verify --release test_chunk_prove_verify
 
 test-e2e-prove:
-	@cargo test --features prove_verify --release test_e2e_prove_verify
+	@cargo test --release -p integration --test e2e_tests test_e2e_prove_verify
 
 test-pi:
 	@cargo test --features prove_verify --release test_batch_pi

README.md

@@ -16,12 +16,6 @@ cargo install svm-rs
 svm install 0.8.19
 ```
 
-Fetch git-submodule of test traces
-```shell
-git submodule init
-git submodule update --checkout
-```
-
 Download all setup params, degree `20`, `24` and `26` are used in [config](https://github.com/scroll-tech/scroll-prover/tree/main/integration/configs).
 Could only download params of degree `26`, but it may affect performance (when downsizing params).
 ```shell
@@ -48,12 +42,6 @@ And there are other tests:
 
 Could use the following command to run binaries locally.
 
-If run into linking issues you may need to run
-```shell
-cp `find ./target/release/ | grep libzktrie.so` /usr/local/lib/
-```
-To move the zktrielib into a path where your linker could locate it.
-
 Run zkevm prover to generate chunk proof (work directory is `./integration`)
 ```shell
 cargo build --release --bin zkevm_prove
@@ -80,17 +68,6 @@ Could specify arguments as
 cargo run --release --bin zkevm_verify -- --params=params --proof=proof_data
 ```
 
-### Scripts
-
-- If you have read access for DB, could run command to generate full-proof for batch tests:
-```
-export DB_HOST=
-export DB_USER=
-export DB_NAME=
-
-sh scripts/gen_full_chunk_proofs.sh BATCH_INDEX
-```
-
 ### Dockers
 
 - `docker/chunk-prover` is used to build and run GPU chunk-prover.

bin/src/zkevm_prove.rs

@@ -2,6 +2,7 @@ use clap::Parser;
 use prover::{
     utils::{get_block_trace_from_file, init_env_and_log},
     zkevm::Prover,
+    ChunkProvingTask,
 };
 use std::{env, fs, path::PathBuf, time::Instant};
 
@@ -32,24 +33,29 @@ fn main() {
     let args = Args::parse();
     let mut prover = Prover::from_dirs(&args.params_path, &args.assets_path);
 
-    let mut traces = Vec::new();
+    let mut block_traces = Vec::new();
     let trace_path = PathBuf::from(&args.trace_path);
     if trace_path.is_dir() {
         for entry in fs::read_dir(trace_path).unwrap() {
             let path = entry.unwrap().path();
             if path.is_file() && path.to_str().unwrap().ends_with(".json") {
                 let block_trace = get_block_trace_from_file(path.to_str().unwrap());
-                traces.push(block_trace);
+                block_traces.push(block_trace);
             }
         }
     } else {
         let block_trace = get_block_trace_from_file(trace_path.to_str().unwrap());
-        traces.push(block_trace);
+        block_traces.push(block_trace);
     }
 
     let now = Instant::now();
     prover
-        .gen_chunk_proof(traces, Some("zkevm"), None, Some(&output_dir))
+        .gen_chunk_proof(
+            ChunkProvingTask { block_traces },
+            Some("zkevm"),
+            None,
+            Some(&output_dir),
+        )
         .expect("cannot generate chunk snark");
     log::info!(
         "finish generating chunk snark, elapsed: {:?}",

integration/Cargo.toml

@@ -11,15 +11,13 @@ itertools.workspace = true
 log.workspace = true
 log4rs.workspace = true
 rand.workspace = true
+serde.workspace = true
+serde_derive.workspace = true
 
 halo2_proofs.workspace = true
 prover.workspace = true
 zkevm-circuits.workspace = true
 
-[dev-dependencies]
-serde.workspace = true
-serde_derive.workspace = true
-
 [features]
 default = ["prove_verify"]
 prove_verify = []

integration/src/test_util.rs

@@ -5,8 +5,10 @@ use prover::{
 };
 
 mod capacity_checker;
-pub mod mock_plonk;
 mod proof;
+mod types;
+
+pub use prover::types::BatchProvingTask;
 
 pub use capacity_checker::{
     ccc_as_signer, ccc_by_chunk, prepare_circuit_capacity_checker, pretty_print_row_usage,
@@ -20,46 +22,30 @@ pub use proof::{
 pub const ASSETS_DIR: &str = "./test_assets";
 pub const PARAMS_DIR: &str = "./params";
 
-pub fn load_block_traces_for_test() -> (Vec<String>, Vec<BlockTrace>) {
+pub fn load_chunk_for_test() -> (Vec<String>, Vec<BlockTrace>) {
     let trace_path: String = read_env_var(
         "TRACE_PATH",
         "./tests/extra_traces/batch_495/chunk_495/block_8802.json".to_string(),
     );
+    load_chunk(&trace_path)
+}
+
+pub fn load_chunk(trace_path: &str) -> (Vec<String>, Vec<BlockTrace>) {
     let paths: Vec<String> = if !std::fs::metadata(&trace_path).unwrap().is_dir() {
-        vec![trace_path]
+        vec![trace_path.to_string()]
     } else {
-        load_chunk_traces(&trace_path).0
+        // Nested dirs are not allowed
+        let mut file_names: Vec<String> = glob(&format!("{trace_path}/*.json"))
+            .unwrap()
+            .map(|p| p.unwrap().to_str().unwrap().to_string())
+            .collect();
+        file_names.sort_by_key(|s| {
+            // Remove the ".json" suffix and parse the remaining part as an integer
+            s.trim_end_matches(".json").parse::<u32>().unwrap()
+        });
+        file_names
     };
     log::info!("test cases traces: {:?}", paths);
     let traces: Vec<_> = paths.iter().map(get_block_trace_from_file).collect();
     (paths, traces)
 }
-
-fn load_chunk_traces(chunk_dir: &str) -> (Vec<String>, Vec<BlockTrace>) {
-    // Nested dirs are not allowed
-    let file_names: Vec<String> = glob(&format!("{chunk_dir}/*.json"))
-        .unwrap()
-        .map(|p| p.unwrap().to_str().unwrap().to_string())
-        .collect();
-    log::info!("test chunk with {:?}", file_names);
-    let mut names_and_traces = file_names
-        .into_iter()
-        .map(|trace_path| {
-            let trace: BlockTrace = get_block_trace_from_file(trace_path.clone());
-            (
-                trace_path,
-                trace.clone(),
-                trace.header.number.unwrap().as_u64(),
-            )
-        })
-        .collect::<Vec<_>>();
-    names_and_traces.sort_by(|a, b| a.2.cmp(&b.2));
-    log::info!(
-        "sorted: {:?}",
-        names_and_traces
-            .iter()
-            .map(|(f, _, _)| f.clone())
-            .collect::<Vec<String>>()
-    );
-    names_and_traces.into_iter().map(|(f, t, _)| (f, t)).unzip()
-}