Skip to content

Commit

Permalink
refactor ccc codes (#326)
Browse files Browse the repository at this point in the history 
* refactor ccc codes

* clippy
  • Loading branch information
@lispc
lispc authored Jun 23, 2024
1 parent 1bb29e1 commit 1074406
Show file tree
Hide file tree
Showing 3 changed files with 94 additions and 47 deletions.
20 changes: 9 additions & 11 deletions bin/src/chain_prover.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
// For production prover, see https://github.com/scroll-tech/scroll/tree/develop/prover

use integration::capacity_checker::{
prepare_circuit_capacity_checker, run_circuit_capacity_checker,
prepare_circuit_capacity_checker, run_circuit_capacity_checker, CCCMode,
};
use prover::{
utils::init_env_and_log,
Expand Down Expand Up @@ -157,10 +157,7 @@ impl ChunkBuilder {
// Construct chunk myself
async fn prove_by_block(l2geth: &l2geth_client::Client, begin_block: i64, end_block: i64) {
let mut chunk_builder = ChunkBuilder::new();
let force_curie = true;
if force_curie {
chunk_builder.block_limit = Some(1);
}
chunk_builder.block_limit = Some(1);
let mut batch_builder = BatchBuilder::new();
let (begin_block, end_block) = if begin_block == 0 && end_block == 0 {
// Blocks within last 24 hours
Expand All @@ -174,7 +171,7 @@ async fn prove_by_block(l2geth: &l2geth_client::Client, begin_block: i64, end_bl
let mut batch_begin_block = begin_block;
for block_num in begin_block..=end_block {
let trace = l2geth
.get_block_trace_by_num(block_num, force_curie)
.get_block_trace_by_num(block_num, false)
.await
.unwrap_or_else(|e| {
panic!("chain_prover: failed to request l2geth block-trace API for block-{block_num}: {e}")
Expand All @@ -187,7 +184,7 @@ async fn prove_by_block(l2geth: &l2geth_client::Client, begin_block: i64, end_bl
100.0 * (block_num - begin_block + 1) as f32 / (end_block - begin_block + 1) as f32
);
if let Some(chunk) = chunk_builder.add(trace) {
prove_chunk(0, 0, chunk.clone());
prove_chunk(0, chunk[0].header.number.unwrap().as_u64(), chunk.clone());
let fast = false;
let chunk_info = if fast {
unimplemented!("uncomment below");
Expand Down Expand Up @@ -232,7 +229,7 @@ fn padding_chunk(chunks: &mut Vec<ChunkInfo>) {
}
}

fn prove_chunk(batch_id: i64, chunk_id: i64, block_traces: Vec<BlockTrace>) -> Option<ChunkProof> {
fn prove_chunk(batch_id: u64, chunk_id: u64, block_traces: Vec<BlockTrace>) -> Option<ChunkProof> {
let total_gas: u64 = block_traces
.iter()
.map(|b| b.header.gas_used.as_u64())
Expand All @@ -247,7 +244,8 @@ fn prove_chunk(batch_id: i64, chunk_id: i64, block_traces: Vec<BlockTrace>) -> O
return None;
}
if env::var("CIRCUIT").unwrap_or_default() == "ccc" {
run_circuit_capacity_checker(batch_id, chunk_id, &block_traces);
let ccc_modes = [CCCMode::Optimal];
run_circuit_capacity_checker(batch_id, chunk_id, &block_traces, &ccc_modes);
return None;
}

Expand Down Expand Up @@ -281,7 +279,7 @@ async fn prove_by_batch(

let mut chunk_proofs = vec![];
for chunk in chunks.unwrap() {
let chunk_id = chunk.index;
let chunk_id = chunk.index as u64;
log::info!("chain_prover: handling chunk {:?}", chunk_id);

let mut block_traces: Vec<BlockTrace> = vec![];
Expand All @@ -296,7 +294,7 @@ async fn prove_by_batch(
block_traces.push(trace);
}

let chunk_proof = prove_chunk(batch_id, chunk_id, block_traces);
let chunk_proof = prove_chunk(batch_id as u64, chunk_id, block_traces);

if let Some(chunk_proof) = chunk_proof {
chunk_proofs.push(chunk_proof);
Expand Down
111 changes: 77 additions & 34 deletions integration/src/capacity_checker.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,14 @@ use prover::{
};
use std::time::Duration;

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CCCMode {
Optimal,
Siger,
FollowerLight,
FollowerFull,
}

pub fn prepare_circuit_capacity_checker() {
// Force evm_circuit::param::EXECUTION_STATE_HEIGHT_MAP to be initialized.
let mulmod_height = ExecutionState::MULMOD.get_step_height();
Expand All @@ -18,31 +26,49 @@ pub fn prepare_circuit_capacity_checker() {

// Return average ccc time for each tx.
pub fn run_circuit_capacity_checker(
batch_id: i64,
chunk_id: i64,
batch_id: u64,
chunk_id: u64,
block_traces: &[BlockTrace],
) -> Duration {
let optimal = ccc_by_chunk(batch_id, chunk_id, block_traces);
let signer = ccc_as_signer(chunk_id, block_traces);
let follower_light = ccc_as_follower_light(chunk_id, block_traces);
let follower_full = ccc_as_follower_full(chunk_id, block_traces);
modes: &[CCCMode],
) -> Option<Duration> {
let results = modes
.iter()
.map(|mode| {
(
*mode,
match mode {
CCCMode::Optimal => ccc_by_chunk(batch_id, chunk_id, block_traces),
CCCMode::Siger => ccc_as_signer(chunk_id, block_traces),
CCCMode::FollowerLight => ccc_as_follower_light(chunk_id, block_traces),
CCCMode::FollowerFull => ccc_as_follower_full(chunk_id, block_traces),
},
)
})
.collect_vec();

for (tag, r) in [
("signer", signer.0),
("follower_light", follower_light.0),
("follower_full", follower_full.0),
] {
compare_ccc_results(chunk_id, &optimal, &r, tag);
if results.len() > 1 {
for idx in 1..results.len() {
compare_ccc_results(
chunk_id,
&results[0].1 .0,
results[0].0,
&results[idx].1 .0,
results[idx].0,
);
}
}

signer.1
results
.into_iter()
.find(|(mode, _)| *mode == CCCMode::Siger)
.map(|(_, (_, t))| t)
}

/// print analyze results
pub fn pretty_print_row_usage(
rows: &RowUsage,
block_traces: &[BlockTrace],
chunk_id: i64,
chunk_id: u64,
mode: &str,
) {
let gas_total: u64 = block_traces
Expand Down Expand Up @@ -112,6 +138,7 @@ fn ccc_block_tx_by_tx(checker: &mut CircuitCapacityChecker, block_idx: usize, bl
storage_trace: block.tx_storage_trace[tx_idx].clone(),
chain_id: block.chain_id,
coinbase: block.coinbase.clone(),
codes: block.codes.clone(),
header: block.header.clone(),
start_l1_queue_index: block.start_l1_queue_index,
..Default::default()
Expand All @@ -124,7 +151,7 @@ fn ccc_block_tx_by_tx(checker: &mut CircuitCapacityChecker, block_idx: usize, bl

// Return row-usage and average ccc time for each tx.
fn get_ccc_result_of_chunk(
chunk_id: i64,
chunk_id: u64,
blocks: &[BlockTrace],
by_block: bool, // by block instead of by tx
norm: bool,
Expand Down Expand Up @@ -195,7 +222,7 @@ fn get_ccc_result_of_chunk(

#[allow(dead_code)]
fn get_ccc_result_by_whole_block(
chunk_id: i64,
chunk_id: u64,
light_mode: bool,
blocks: &[BlockTrace],
) -> RowUsage {
Expand All @@ -221,46 +248,62 @@ fn get_ccc_result_by_whole_block(
ccc_result
}

fn compare_ccc_results(chunk_id: i64, base: &RowUsage, estimate: &RowUsage, tag: &str) {
for (b, e) in base
fn compare_ccc_results(
chunk_id: u64,
lhs: &RowUsage,
lhs_mode: CCCMode,
rhs: &RowUsage,
rhs_mode: CCCMode,
) {
for (l, r) in lhs
.row_usage_details
.iter()
.zip_eq(estimate.row_usage_details.iter())
.chain(std::iter::once((&bottleneck(base), &bottleneck(estimate))))
.zip_eq(rhs.row_usage_details.iter())
.chain(std::iter::once((&bottleneck(lhs), &bottleneck(rhs))))
{
log::info!(
"chunk {chunk_id}: opt {} {} vs {tag} {} {}. over estimate ratio {}",
b.name,
b.row_number,
e.name,
e.row_number,
e.row_number as f64 / b.row_number as f64
"chunk {chunk_id}: {lhs_mode:?} {} {} vs {rhs_mode:?} {} {}. r/l: {}",
l.name,
l.row_number,
r.name,
r.row_number,
r.row_number as f64 / l.row_number as f64
);
// FIXME the "+1", bytecode
assert!(e.row_number + 1 >= b.row_number);
if lhs_mode == CCCMode::Optimal {
// FIXME the "+1", bytecode
assert!(r.row_number + 1 >= l.row_number);
}
}
}

/// most accurate, optimal
pub fn ccc_by_chunk(batch_id: i64, chunk_id: i64, block_traces: &[BlockTrace]) -> RowUsage {
pub fn ccc_by_chunk(
batch_id: u64,
chunk_id: u64,
block_traces: &[BlockTrace],
) -> (RowUsage, Duration) {
log::info!("ccc_by_chunk: run ccc for batch-{batch_id} chunk-{chunk_id}");

let witness_block = block_traces_to_witness_block(Vec::from(block_traces)).unwrap();
let rows = calculate_row_usage_of_witness_block(&witness_block).unwrap();
let row_usage = RowUsage::from_row_usage_details(rows);
pretty_print_row_usage(&row_usage, block_traces, chunk_id, "chunk-opt");
row_usage

// TODO: finish this
let avg_ccc_time_per_tx = Default::default();

(row_usage, avg_ccc_time_per_tx)
}

pub fn ccc_as_signer(chunk_id: i64, blocks: &[BlockTrace]) -> (RowUsage, Duration) {
pub fn ccc_as_signer(chunk_id: u64, blocks: &[BlockTrace]) -> (RowUsage, Duration) {
get_ccc_result_of_chunk(chunk_id, blocks, false, false, true, "chunk-signer")
}

/// current stats inside db
pub fn ccc_as_follower_light(chunk_id: i64, blocks: &[BlockTrace]) -> (RowUsage, Duration) {
pub fn ccc_as_follower_light(chunk_id: u64, blocks: &[BlockTrace]) -> (RowUsage, Duration) {
get_ccc_result_of_chunk(chunk_id, blocks, true, false, true, "chunk-f-l")
}

pub fn ccc_as_follower_full(chunk_id: i64, blocks: &[BlockTrace]) -> (RowUsage, Duration) {
pub fn ccc_as_follower_full(chunk_id: u64, blocks: &[BlockTrace]) -> (RowUsage, Duration) {
get_ccc_result_of_chunk(chunk_id, blocks, true, false, false, "chunk-f-f")
}
10 changes: 8 additions & 2 deletions integration/tests/unit_tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

use integration::{
capacity_checker::{
ccc_as_signer, prepare_circuit_capacity_checker, run_circuit_capacity_checker,
ccc_as_signer, prepare_circuit_capacity_checker, run_circuit_capacity_checker, CCCMode,
},
test_util::load_chunk_for_test,
};
Expand Down Expand Up @@ -78,7 +78,13 @@ fn test_capacity_checker() {
let batch_id = 0;
let chunk_id = 0;
let avg_each_tx_time = if full {
run_circuit_capacity_checker(batch_id, chunk_id, &block_traces)
let ccc_modes = [
CCCMode::Optimal,
CCCMode::Siger,
CCCMode::FollowerLight,
CCCMode::FollowerFull,
];
run_circuit_capacity_checker(batch_id, chunk_id, &block_traces, &ccc_modes).unwrap()
} else {
ccc_as_signer(chunk_id, &block_traces).1
};
Expand Down

0 comments on commit 1074406

Please sign in to comment.