From 3660408a1cedacfa7281b2e12b2e90b841a8242a Mon Sep 17 00:00:00 2001 From: Sai <135601871+sai-deng@users.noreply.github.com> Date: Mon, 7 Oct 2024 17:17:13 +0000 Subject: [PATCH] Enables optional verification of Keccak tables (#657) * refactoring * fix clippy * fix clippy * wip * wip * wip * cleanup * prove one segment * polish the code * cleanup * Revert "cleanup" This reverts commit f63363b6ac07fd8d4c875223ce3997d6835e3264. * add [ignore] * add challenger state check * add more checks * cleanup * add empty keccak tables bool * enable no keccak recursion * refactor * fix tests * rename * fix pis * fix * done * polish the comments * fix ci * add comments * fix ci * update tests * fix ci * fix ci * fix ci * empty payload and no keccak op check * refactor * Update evm_arithmetization/src/fixed_recursive_verifier.rs Co-authored-by: Robin Salen <30937548+Nashtare@users.noreply.github.com> * Update evm_arithmetization/src/fixed_recursive_verifier.rs Co-authored-by: Robin Salen <30937548+Nashtare@users.noreply.github.com> * empty payload and no keccak op check * fmt * fix test * fix ci with "cdk_erigon" * fix ci error with "cdk_erigon" * enable test * Update evm_arithmetization/src/fixed_recursive_verifier.rs Co-authored-by: Linda Guiga <101227802+LindaGuiga@users.noreply.github.com> * address comments * sync changes into get_challenges * wip * fix * fix * fix tests * fix * address comments * common_date -> common_data --------- Co-authored-by: Robin Salen <30937548+Nashtare@users.noreply.github.com> Co-authored-by: Linda Guiga <101227802+LindaGuiga@users.noreply.github.com> --- .../src/fixed_recursive_verifier.rs | 120 ++++++++++++------ evm_arithmetization/src/generation/mod.rs | 17 ++- evm_arithmetization/src/get_challenges.rs | 37 ++++-- evm_arithmetization/src/proof.rs | 4 + evm_arithmetization/src/prover.rs | 31 ++++- evm_arithmetization/src/verifier.rs | 10 +- 6 files changed, 159 insertions(+), 60 deletions(-) diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index f3c9fa7b6..746f63f44 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -23,7 +23,7 @@ use plonky2::plonk::circuit_data::{ use plonky2::plonk::config::{AlgebraicHasher, GenericConfig, GenericHashOut}; use plonky2::plonk::proof::{ProofWithPublicInputs, ProofWithPublicInputsTarget}; use plonky2::recursion::cyclic_recursion::check_cyclic_proof_verifier_data; -use plonky2::recursion::dummy_circuit::cyclic_base_proof; +use plonky2::recursion::dummy_circuit::{cyclic_base_proof, dummy_circuit, dummy_proof}; use plonky2::util::serialization::{ Buffer, GateSerializer, IoResult, Read, WitnessGeneratorSerializer, Write, }; @@ -1916,30 +1916,57 @@ where let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { - let stark_proof = &all_proof.multi_proof.stark_proofs[table]; - let original_degree_bits = stark_proof.proof.recover_degree_bits(config); let table_circuits = &self.by_table[table]; - let shrunk_proof = table_circuits - .by_stark_size - .get(&original_degree_bits) - .ok_or_else(|| { - anyhow!(format!( - "Missing preprocessed circuits for {:?} table with size {}.", - Table::all()[table], - original_degree_bits, - )) - })? - .shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; - let index_verifier_data = table_circuits - .by_stark_size - .keys() - .position(|&size| size == original_degree_bits) - .unwrap(); - root_inputs.set_target( - self.root.index_verifier_data[table], - F::from_canonical_usize(index_verifier_data), - ); - root_inputs.set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); + if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { + // generate and set a dummy `index_verifier_data` and `proof_with_pis` + let index_verifier_data = + table_circuits.by_stark_size.keys().min().ok_or_else(|| { + anyhow::format_err!("No valid size in shrinking circuits") + })?; + root_inputs.set_target( + self.root.index_verifier_data[table], + F::from_canonical_usize(*index_verifier_data), + ); + let table_circuit = table_circuits + .by_stark_size + .get(index_verifier_data) + .ok_or_else(|| anyhow::format_err!("No valid size in shrinking circuits"))? + .shrinking_wrappers + .last() + .ok_or_else(|| anyhow::format_err!("No shrinking circuits"))?; + let dummy_circuit: CircuitData = + dummy_circuit(&table_circuit.circuit.common); + let dummy_pis = HashMap::new(); + let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) + .expect("Unable to generate dummy proofs"); + root_inputs + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); + } else { + let stark_proof = &all_proof.multi_proof.stark_proofs[table]; + let original_degree_bits = stark_proof.proof.recover_degree_bits(config); + let shrunk_proof = table_circuits + .by_stark_size + .get(&original_degree_bits) + .ok_or_else(|| { + anyhow!(format!( + "Missing preprocessed circuits for {:?} table with size {}.", + Table::all()[table], + original_degree_bits, + )) + })? + .shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; + let index_verifier_data = table_circuits + .by_stark_size + .keys() + .position(|&size| size == original_degree_bits) + .unwrap(); + root_inputs.set_target( + self.root.index_verifier_data[table], + F::from_canonical_usize(index_verifier_data), + ); + root_inputs + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); + } check_abort_signal(abort_signal.clone())?; } @@ -1958,8 +1985,7 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; - // TODO(sdeng): Set to false when this segment contains no Keccak operations. - root_inputs.set_bool_target(self.root.use_keccak_tables, true); + root_inputs.set_bool_target(self.root.use_keccak_tables, all_proof.use_keccak_tables); let root_proof = self.root.circuit.prove(root_inputs)?; @@ -2033,16 +2059,36 @@ where for table in 0..NUM_TABLES { let (table_circuit, index_verifier_data) = &table_circuits[table]; + if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { + root_inputs.set_target( + self.root.index_verifier_data[table], + F::from_canonical_u8(*index_verifier_data), + ); + // generate and set a dummy `proof_with_pis` + let common_data = &table_circuit + .shrinking_wrappers + .last() + .ok_or_else(|| anyhow::format_err!("No shrinking circuits"))? + .circuit + .common; + let dummy_circuit: CircuitData = dummy_circuit(common_data); + let dummy_pis = HashMap::new(); + let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) + .expect("Unable to generate dummy proofs"); + root_inputs + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); + } else { + let stark_proof = &all_proof.multi_proof.stark_proofs[table]; - let stark_proof = &all_proof.multi_proof.stark_proofs[table]; - - let shrunk_proof = - table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; - root_inputs.set_target( - self.root.index_verifier_data[table], - F::from_canonical_u8(*index_verifier_data), - ); - root_inputs.set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); + let shrunk_proof = + table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; + root_inputs.set_target( + self.root.index_verifier_data[table], + F::from_canonical_u8(*index_verifier_data), + ); + root_inputs + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); + } check_abort_signal(abort_signal.clone())?; } @@ -2061,8 +2107,7 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; - // TODO(sdeng): Set to false when this segment contains no Keccak operations. - root_inputs.set_bool_target(self.root.use_keccak_tables, true); + root_inputs.set_bool_target(self.root.use_keccak_tables, all_proof.use_keccak_tables); let root_proof = self.root.circuit.prove(root_inputs)?; @@ -3103,7 +3148,6 @@ mod tests { type C = PoseidonGoldilocksConfig; #[test] - #[ignore] fn test_segment_proof_generation_without_keccak() -> anyhow::Result<()> { init_logger(); diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 19ea55039..a4994f686 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -486,7 +486,11 @@ fn get_all_memory_address_and_values(memory_before: &MemoryState) -> Vec<(Memory res } -type TablesWithPVsAndFinalMem = ([Vec>; NUM_TABLES], PublicValues); +pub struct TablesWithPVs { + pub tables: [Vec>; NUM_TABLES], + pub use_keccak_tables: bool, + pub public_values: PublicValues, +} pub fn generate_traces, const D: usize>( all_stark: &AllStark, @@ -494,7 +498,7 @@ pub fn generate_traces, const D: usize>( config: &StarkConfig, segment_data: &mut GenerationSegmentData, timing: &mut TimingTree, -) -> anyhow::Result> { +) -> anyhow::Result> { let mut state = GenerationState::::new_with_segment_data(inputs, segment_data) .map_err(|err| anyhow!("Failed to parse all the initial prover inputs: {:?}", err))?; @@ -579,6 +583,8 @@ pub fn generate_traces, const D: usize>( mem_after: MemCap::default(), }; + let use_keccak_tables = !state.traces.keccak_inputs.is_empty(); + let tables = timed!( timing, "convert trace data to tables", @@ -591,7 +597,12 @@ pub fn generate_traces, const D: usize>( timing ) ); - Ok((tables, public_values)) + + Ok(TablesWithPVs { + tables, + use_keccak_tables, + public_values, + }) } fn simulate_cpu( diff --git a/evm_arithmetization/src/get_challenges.rs b/evm_arithmetization/src/get_challenges.rs index 1e6410353..ca1f06c62 100644 --- a/evm_arithmetization/src/get_challenges.rs +++ b/evm_arithmetization/src/get_challenges.rs @@ -254,6 +254,7 @@ pub mod testing { use starky::lookup::{get_grand_product_challenge_set, GrandProductChallengeSet}; use starky::proof::StarkProofChallenges; + use crate::all_stark::KECCAK_TABLES_INDICES; use crate::get_challenges::observe_public_values; use crate::proof::*; use crate::witness::errors::ProgramError; @@ -262,7 +263,7 @@ pub mod testing { /// Randomness for all STARKs. pub(crate) struct AllProofChallenges, const D: usize> { /// Randomness used in each STARK proof. - pub stark_challenges: [StarkProofChallenges; NUM_TABLES], + pub stark_challenges: [Option>; NUM_TABLES], /// Randomness used for cross-table lookups. It is shared by all STARKs. pub ctl_challenges: GrandProductChallengeSet, } @@ -277,8 +278,20 @@ pub mod testing { let stark_proofs = &self.multi_proof.stark_proofs; - for proof in stark_proofs { - challenger.observe_cap(&proof.proof.trace_cap); + for (i, proof) in stark_proofs.iter().enumerate() { + if KECCAK_TABLES_INDICES.contains(&i) && !self.use_keccak_tables { + // Observe zero merkle caps when skipping Keccak tables. + let zero_merkle_cap = proof + .proof + .trace_cap + .flatten() + .iter() + .map(|_| F::ZERO) + .collect::>(); + challenger.observe_elements(&zero_merkle_cap); + } else { + challenger.observe_cap(&proof.proof.trace_cap); + } } observe_public_values::(&mut challenger, &self.public_values)?; @@ -288,13 +301,17 @@ pub mod testing { Ok(AllProofChallenges { stark_challenges: core::array::from_fn(|i| { - challenger.compact(); - stark_proofs[i].proof.get_challenges( - &mut challenger, - Some(&ctl_challenges), - true, - config, - ) + if KECCAK_TABLES_INDICES.contains(&i) && !self.use_keccak_tables { + None + } else { + challenger.compact(); + Some(stark_proofs[i].proof.get_challenges( + &mut challenger, + Some(&ctl_challenges), + true, + config, + )) + } }), ctl_challenges, }) diff --git a/evm_arithmetization/src/proof.rs b/evm_arithmetization/src/proof.rs index ed3e20ee7..304072aed 100644 --- a/evm_arithmetization/src/proof.rs +++ b/evm_arithmetization/src/proof.rs @@ -30,6 +30,10 @@ pub struct AllProof, C: GenericConfig, co pub multi_proof: MultiProof, /// Public memory values used for the recursive proofs. pub public_values: PublicValues, + /// A flag indicating whether the Keccak and KeccakSponge tables contain + /// only padding values (i.e., no meaningful data). This is set to false + /// when no actual Keccak operations were performed. + pub use_keccak_tables: bool, } impl, C: GenericConfig, const D: usize> AllProof { diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index 1c28362c0..fdc5eca20 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -20,7 +20,7 @@ use starky::proof::{MultiProof, StarkProofWithMetadata}; use starky::prover::prove_with_commitment; use starky::stark::Stark; -use crate::all_stark::{AllStark, Table, NUM_TABLES}; +use crate::all_stark::{AllStark, Table, KECCAK_TABLES_INDICES, NUM_TABLES}; use crate::cpu::kernel::aggregator::KERNEL; use crate::generation::{generate_traces, GenerationInputs, TrimmedGenerationInputs}; use crate::get_challenges::observe_public_values; @@ -47,7 +47,7 @@ where timed!(timing, "build kernel", Lazy::force(&KERNEL)); - let (traces, mut public_values) = timed!( + let mut tables_with_pvs = timed!( timing, "generate all traces", generate_traces(all_stark, &inputs, config, segment_data, timing)? @@ -58,8 +58,9 @@ where let proof = prove_with_traces( all_stark, config, - traces, - &mut public_values, + tables_with_pvs.tables, + tables_with_pvs.use_keccak_tables, + &mut tables_with_pvs.public_values, timing, abort_signal, )?; @@ -72,6 +73,7 @@ pub(crate) fn prove_with_traces( all_stark: &AllStark, config: &StarkConfig, trace_poly_values: [Vec>; NUM_TABLES], + use_keccak_tables: bool, public_values: &mut PublicValues, timing: &mut TimingTree, abort_signal: Option>, @@ -114,8 +116,14 @@ where .map(|c| c.merkle_tree.cap.clone()) .collect::>(); let mut challenger = Challenger::::new(); - for cap in &trace_caps { - challenger.observe_cap(cap); + for (i, cap) in trace_caps.iter().enumerate() { + if KECCAK_TABLES_INDICES.contains(&i) && !use_keccak_tables { + // Observe zero merkle caps when skipping Keccak tables. + let zero_merkle_cap = cap.flatten().iter().map(|_| F::ZERO).collect::>(); + challenger.observe_elements(&zero_merkle_cap); + } else { + challenger.observe_cap(cap); + } } observe_public_values::(&mut challenger, public_values) @@ -143,6 +151,7 @@ where config, &trace_poly_values, trace_commitments, + use_keccak_tables, ctl_data_per_table, &mut challenger, &ctl_challenges, @@ -206,6 +215,7 @@ where ctl_challenges, }, public_values: public_values.clone(), + use_keccak_tables, }) } @@ -229,6 +239,7 @@ fn prove_with_commitments( config: &StarkConfig, trace_poly_values: &[Vec>; NUM_TABLES], trace_commitments: Vec>, + use_keccak_tables: bool, ctl_data_per_table: [CtlData; NUM_TABLES], challenger: &mut Challenger, ctl_challenges: &GrandProductChallengeSet, @@ -262,8 +273,16 @@ where let (arithmetic_proof, _) = prove_table!(arithmetic_stark, Table::Arithmetic); let (byte_packing_proof, _) = prove_table!(byte_packing_stark, Table::BytePacking); let (cpu_proof, _) = prove_table!(cpu_stark, Table::Cpu); + let challenger_after_cpu = challenger.clone(); + // TODO(sdeng): Keccak proofs are still required for CTLs, etc. Refactor the + // code and remove the unnecessary parts. let (keccak_proof, _) = prove_table!(keccak_stark, Table::Keccak); let (keccak_sponge_proof, _) = prove_table!(keccak_sponge_stark, Table::KeccakSponge); + if !use_keccak_tables { + // We need to connect the challenger state of CPU and Logic tables when the + // Keccak tables are not in use. + *challenger = challenger_after_cpu; + } let (logic_proof, _) = prove_table!(logic_stark, Table::Logic); let (memory_proof, _) = prove_table!(memory_stark, Table::Memory); let (mem_before_proof, mem_before_cap) = prove_table!(mem_before_stark, Table::MemBefore); diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index 1c3f090c5..947516801 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -199,7 +199,9 @@ pub mod testing { verify_stark_proof_with_challenges( $stark, &stark_proofs[*$table].proof, - &stark_challenges[*$table], + &stark_challenges[*$table] + .as_ref() + .expect("Missing challenges"), Some(&ctl_vars_per_table[*$table]), &[], config, @@ -210,8 +212,10 @@ pub mod testing { verify_table!(arithmetic_stark, Table::Arithmetic); verify_table!(byte_packing_stark, Table::BytePacking); verify_table!(cpu_stark, Table::Cpu); - verify_table!(keccak_stark, Table::Keccak); - verify_table!(keccak_sponge_stark, Table::KeccakSponge); + if all_proof.use_keccak_tables { + verify_table!(keccak_stark, Table::Keccak); + verify_table!(keccak_sponge_stark, Table::KeccakSponge); + } verify_table!(logic_stark, Table::Logic); verify_table!(memory_stark, Table::Memory); verify_table!(mem_before_stark, Table::MemBefore);