diff --git a/crates/neo-midnight-bridge/README.md b/crates/neo-midnight-bridge/README.md new file mode 100644 index 00000000..f89e7019 --- /dev/null +++ b/crates/neo-midnight-bridge/README.md @@ -0,0 +1,44 @@ +# neo-midnight-bridge + +Experimental bridge: prove Neo FoldRun validity using Midnight's PLONK/KZG verifier stack. + +## KZG params (BLS12-381 SRS) + +For local tests/benchmarks, this crate can load `midnight-proofs` `ParamsKZG` files from: + +`crates/neo-midnight-bridge/testdata/kzg_params/bls_midnight_2p{k}` + +### Download pre-generated Midnight params + +Midnight publishes pre-generated parameter files under: + +`https://midnight-s3-fileshare-dev-eu-west-1.s3.eu-west-1.amazonaws.com/bls_midnight_2p{k}` + +Example downloads: + +```bash +BASE_URL="https://midnight-s3-fileshare-dev-eu-west-1.s3.eu-west-1.amazonaws.com" +OUT_DIR="crates/neo-midnight-bridge/testdata/kzg_params" + +mkdir -p "$OUT_DIR" + +# Download k=16,17,18 +for k in 16 17 18; do + curl -L --fail -o "$OUT_DIR/bls_midnight_2p${k}" "$BASE_URL/bls_midnight_2p${k}" +done +``` + +Notes: +- Files can be large (e.g. `k=21` is ~400MB). +- Midnight-ledger contains SHA-256 hashes for each `bls_midnight_2p{k}` for integrity checking. + +### Generate params from a Powers-of-Tau transcript + +If you have a Midnight Powers-of-Tau transcript file (raw bytes: G1 powers followed by two G2 +points), you can convert it into `ParamsKZG` files via: + +```bash +cargo run -p neo-midnight-bridge --example params_from_powers_of_tau -- \ + crates/neo-midnight-bridge/testdata/kzg_params [k_max] +``` + diff --git a/crates/neo-midnight-bridge/examples/params_from_powers_of_tau.rs b/crates/neo-midnight-bridge/examples/params_from_powers_of_tau.rs new file mode 100644 index 00000000..0f037367 --- /dev/null +++ b/crates/neo-midnight-bridge/examples/params_from_powers_of_tau.rs @@ -0,0 +1,90 @@ +//! Convert a Midnight Powers-of-Tau transcript into `midnight-proofs` `ParamsKZG` files. +//! +//! This mirrors `external/midnight-ledger/transient-crypto/examples/translate-params.rs`, +//! but lives in this repo so we can generate params for outer compression proofs without +//! relying on Midnight ledger tooling. +//! +//! If you don't have a `powers_of_tau` transcript, you can alternatively download +//! pre-generated `bls_midnight_2p{k}` files and place them under: +//! `crates/neo-midnight-bridge/testdata/kzg_params/`. +//! +//! Usage: +//! cargo run -p neo-midnight-bridge --example params_from_powers_of_tau -- \ +//! [k_max] +//! +//! Output: +//! `/bls_midnight_2p{k}` for k in 0..=k_max (written in RawBytes format). + +use midnight_curves::{serde::SerdeObject, Bls12, G1Affine, G2Affine}; +use midnight_proofs::{poly::kzg::params::ParamsKZG, utils::SerdeFormat}; +use std::path::PathBuf; + +fn floor_log2(mut n: usize) -> u32 { + let mut log = 0u32; + while n > 1 { + n >>= 1; + log += 1; + } + log +} + +fn main() { + let mut args = std::env::args().skip(1); + let pot_path = args + .next() + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("powers_of_tau")); + let out_dir = args + .next() + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(".")); + let k_max_arg = args + .next() + .map(|s| s.parse::().expect("k_max must be an integer")); + + let bytes = std::fs::read(&pot_path).expect("read powers_of_tau"); + let g1_size = G1Affine::uncompressed_size(); + let g2_size = G2Affine::uncompressed_size(); + + assert!(bytes.len() >= 2 * g2_size, "powers_of_tau too short: need >= 2*G2"); + let offset = bytes.len() - 2 * g2_size; + assert_eq!(offset % g1_size, 0, "powers_of_tau length not aligned to G1 size"); + + let g1_count = offset / g1_size; + let k_max_default = floor_log2(g1_count); + let k_max = k_max_arg.unwrap_or(k_max_default); + let needed = 1usize << k_max; + assert!( + g1_count >= needed, + "powers_of_tau has {g1_count} G1 points, but k_max={k_max} needs >= {needed}" + ); + + println!("Reading powers_of_tau from {pot_path:?}"); + println!("G1 points: {g1_count} (max k by length: {k_max_default})"); + println!("Generating ParamsKZG for k_max={k_max} into {out_dir:?}"); + + // Read G1 powers. + let mut g1s = Vec::with_capacity(g1_count); + for chunk in bytes[..offset].chunks(g1_size) { + let p = G1Affine::from_raw_bytes(chunk).expect("decode G1"); + g1s.push(p.into()); + } + + // Read trailing G2 points (beta_g2, g2). + let g2_0 = G2Affine::from_raw_bytes(&bytes[offset..offset + g2_size]).expect("decode G2[0]"); + let g2_1 = G2Affine::from_raw_bytes(&bytes[offset + g2_size..offset + 2 * g2_size]).expect("decode G2[1]"); + + std::fs::create_dir_all(&out_dir).expect("create out_dir"); + + // Build params at k_max, then downsize in-place and write all smaller k. + let mut params = ParamsKZG::::from_parts(k_max, g1s, None, g2_0.into(), g2_1.into()); + for k in (0..=k_max).rev() { + println!("Writing k={k}"); + params.downsize(k); + let out_path = out_dir.join(format!("bls_midnight_2p{k}")); + let mut f = std::fs::File::create(&out_path).expect("create output file"); + params + .write_custom(&mut f, SerdeFormat::RawBytes) + .expect("write ParamsKZG"); + } +} diff --git a/crates/neo-midnight-bridge/src/relations.rs b/crates/neo-midnight-bridge/src/relations.rs index c735b79e..a2be8fc6 100644 --- a/crates/neo-midnight-bridge/src/relations.rs +++ b/crates/neo-midnight-bridge/src/relations.rs @@ -18,6 +18,24 @@ use midnight_zk_stdlib::{Relation, ZkStdLib, ZkStdLibArch}; use serde::{Deserialize, Serialize}; use std::io::{Read, Write}; +fn write_relation_len_prefixed(writer: &mut W, value: &T) -> std::io::Result<()> { + let bytes = bincode::serialize(value).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + let len = u32::try_from(bytes.len()) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "relation too large"))?; + writer.write_all(&len.to_le_bytes())?; + writer.write_all(&bytes)?; + Ok(()) +} + +fn read_relation_len_prefixed Deserialize<'de>>(reader: &mut R) -> std::io::Result { + let mut len_bytes = [0u8; 4]; + reader.read_exact(&mut len_bytes)?; + let len = u32::from_le_bytes(len_bytes) as usize; + let mut bytes = vec![0u8; len]; + reader.read_exact(&mut bytes)?; + bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) +} + /// Public statement: `z = x * y (mod p)` over Goldilocks. /// /// This is a minimal end-to-end “Option B” sanity check: @@ -47,14 +65,11 @@ impl Relation for GoldilocksMulRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -121,14 +136,11 @@ impl Relation for SumcheckSingleRoundRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -230,14 +242,11 @@ impl Relation for PiCcsSumcheckRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -333,14 +342,11 @@ impl Relation for PiCcsSumcheckNcRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -454,14 +460,11 @@ impl Relation for PiCcsSumcheckPublicRoundsRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -663,14 +666,11 @@ impl Relation for PiCcsFeTerminalK1Relation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -799,14 +799,11 @@ impl Relation for PiCcsFeTerminalRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -1026,14 +1023,11 @@ impl Relation for PiCcsFeChunkRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -1227,14 +1221,11 @@ impl Relation for PiCcsFeTerminalAggregateRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -1406,7 +1397,7 @@ pub struct PiCcsFeChunkAggSumcheckInstance { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct PiCcsFeChunkAggSumcheckWitness { - // Sumcheck witness. + /// FE sumcheck round polynomials (private): `rounds[round_idx][coeff_idx]`. pub rounds: Vec>, // ME input point r for eq(r',r). pub me_inputs_r: Vec, @@ -1465,14 +1456,11 @@ impl Relation for PiCcsFeChunkAggSumcheckRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -1569,6 +1557,17 @@ impl Relation for PiCcsFeChunkAggSumcheckRelation { challenges.push(alloc_k_public(std_lib, layouter, ch)?); } + // Private witness: sumcheck rounds. + let mut rounds: Vec> = Vec::with_capacity(self.n_rounds); + for r in 0..self.n_rounds { + let mut coeffs_r = Vec::with_capacity(self.poly_len); + for j in 0..self.poly_len { + let coeff = witness.as_ref().map(|w| w.rounds[r][j]); + coeffs_r.push(alloc_k_private(std_lib, layouter, coeff)?); + } + rounds.push(coeffs_r); + } + let gamma = alloc_k_public(std_lib, layouter, instance.as_ref().map(|i| i.gamma))?; let mut alpha: Vec = Vec::with_capacity(self.ell_d); @@ -1596,16 +1595,6 @@ impl Relation for PiCcsFeChunkAggSumcheckRelation { let final_sum = alloc_k_public(std_lib, layouter, instance.as_ref().map(|i| i.final_sum))?; // --- Sumcheck --- - let mut rounds: Vec> = Vec::with_capacity(self.n_rounds); - for r in 0..self.n_rounds { - let mut coeffs_r = Vec::with_capacity(self.poly_len); - for j in 0..self.poly_len { - let coeff = witness.as_ref().map(|w| w.rounds[r][j]); - coeffs_r.push(alloc_k_private(std_lib, layouter, coeff)?); - } - rounds.push(coeffs_r); - } - let mut running_sum = initial_sum.clone(); for r in 0..self.n_rounds { sumcheck_round_check(std_lib, layouter, &rounds[r], &running_sum)?; @@ -1796,14 +1785,11 @@ impl Relation for PiCcsNcTerminalK1Relation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -1953,14 +1939,11 @@ impl Relation for PiCcsNcTerminalRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -2139,14 +2122,11 @@ impl Relation for PiCcsNcChunkRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -2305,14 +2285,11 @@ impl Relation for PiCcsNcTerminalAggregateRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -2385,9 +2362,11 @@ impl Relation for PiCcsNcTerminalAggregateRelation { /// Combines: /// - NC sumcheck verification, and -/// - one NC terminal chunk proof that also performs the aggregate check. +/// - (optionally) one NC terminal chunk proof that also performs the aggregate check. /// /// This lets a bundle drop the standalone NC sumcheck proof. +/// +/// Set `count=0` to skip the chunk-binding section (sumcheck + aggregate only). #[derive(Clone, Debug, Serialize, Deserialize)] pub struct PiCcsNcChunkAggSumcheckRelation { // Sumcheck parameters. @@ -2422,9 +2401,11 @@ pub struct PiCcsNcChunkAggSumcheckInstance { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct PiCcsNcChunkAggSumcheckWitness { - // Sumcheck witness. + /// NC sumcheck round polynomials (private): `rounds[round_idx][coeff_idx]`. pub rounds: Vec>, // Digit columns for the designated chunk (length = count, each padded to 2^ell_d). + // + // If `count=0`, this must be empty and the chunk-binding section is skipped. pub y_zcol: Vec>, } @@ -2473,14 +2454,11 @@ impl Relation for PiCcsNcChunkAggSumcheckRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( @@ -2520,11 +2498,7 @@ impl Relation for PiCcsNcChunkAggSumcheckRelation { "PiCcsNcChunkAggSumcheckRelation requires start_exp > 0".into(), )); } - if self.count == 0 { - return Err(Error::Synthesis( - "PiCcsNcChunkAggSumcheckRelation requires count > 0".into(), - )); - } + // `count=0` is allowed: skip chunk-binding and prove sumcheck + aggregate only. if self.n_chunks == 0 { return Err(Error::Synthesis( "PiCcsNcChunkAggSumcheckRelation requires n_chunks > 0".into(), @@ -2562,6 +2536,18 @@ impl Relation for PiCcsNcChunkAggSumcheckRelation { let ch = instance.as_ref().map(|i| i.sumcheck_challenges[r]); challenges.push(alloc_k_public(std_lib, layouter, ch)?); } + + // Private witness: sumcheck rounds. + let mut rounds: Vec> = Vec::with_capacity(self.n_rounds); + for r in 0..self.n_rounds { + let mut coeffs_r = Vec::with_capacity(self.poly_len); + for j in 0..self.poly_len { + let coeff = witness.as_ref().map(|w| w.rounds[r][j]); + coeffs_r.push(alloc_k_private(std_lib, layouter, coeff)?); + } + rounds.push(coeffs_r); + } + let gamma = alloc_k_public(std_lib, layouter, instance.as_ref().map(|i| i.gamma))?; let mut beta_a: Vec = Vec::with_capacity(self.ell_d); @@ -2584,16 +2570,6 @@ impl Relation for PiCcsNcChunkAggSumcheckRelation { let final_sum_nc = alloc_k_public(std_lib, layouter, instance.as_ref().map(|i| i.final_sum_nc))?; // --- Sumcheck --- - let mut rounds: Vec> = Vec::with_capacity(self.n_rounds); - for r in 0..self.n_rounds { - let mut coeffs_r = Vec::with_capacity(self.poly_len); - for j in 0..self.poly_len { - let coeff = witness.as_ref().map(|w| w.rounds[r][j]); - coeffs_r.push(alloc_k_private(std_lib, layouter, coeff)?); - } - rounds.push(coeffs_r); - } - let mut running_sum = initial_sum.clone(); for r in 0..self.n_rounds { sumcheck_round_check(std_lib, layouter, &rounds[r], &running_sum)?; @@ -2604,75 +2580,77 @@ impl Relation for PiCcsNcChunkAggSumcheckRelation { // Split challenges into (s_col_prime, alpha_prime). let (s_col_prime, alpha_prime) = challenges.split_at(self.ell_m); - // --- NC terminal: recompute designated chunk sum --- - - let d_pad = 1usize - .checked_shl(self.ell_d as u32) - .ok_or_else(|| Error::Synthesis("PiCcsNcChunkAggSumcheckRelation: 1< 0 { + let d_pad = 1usize + .checked_shl(self.ell_d as u32) + .ok_or_else(|| Error::Synthesis("PiCcsNcChunkAggSumcheckRelation: 1< = Vec::with_capacity(self.count); - for out_idx in 0..self.count { - let mut y_zcol: Vec = Vec::with_capacity(d_pad); - for i in 0..d_pad { - let v = witness.as_ref().map(|w| w.y_zcol[out_idx][i]); - y_zcol.push(alloc_k_private_u64(std_lib, layouter, v)?); + // g = γ^{start_exp} + let one = k_one(std_lib, layouter)?; + let mut g = one.clone(); + for _ in 0..self.start_exp { + g = k_mul_mod_var(std_lib, layouter, &g, &gamma, K_DELTA_U64)?; } - // y_eval = . - let mut eval_vec = y_zcol; - for a in alpha_prime { - let next_len = eval_vec.len() / 2; - let mut next: Vec = Vec::with_capacity(next_len); - for j in 0..next_len { - let v0 = &eval_vec[2 * j]; - let v1 = &eval_vec[2 * j + 1]; - next.push(k_mle_fold_step(std_lib, layouter, v0, v1, a, K_DELTA_U64)?); + let mut weighted_terms: Vec = Vec::with_capacity(self.count); + for out_idx in 0..self.count { + let mut y_zcol: Vec = Vec::with_capacity(d_pad); + for i in 0..d_pad { + let v = witness.as_ref().map(|w| w.y_zcol[out_idx][i]); + y_zcol.push(alloc_k_private_u64(std_lib, layouter, v)?); } - eval_vec = next; - } - if eval_vec.len() != 1 { - return Err(Error::Synthesis(format!( - "PiCcsNcChunkAggSumcheckRelation: eval_vec len {} != 1 (ell_d={})", - eval_vec.len(), - self.ell_d - ))); - } - let y_eval = eval_vec.first().expect("len checked").clone(); - // range_product(y_eval) = ∏_{t=-(b-1)}^{b-1} (y_eval - t) - let lo = -((self.b as i64) - 1); - let hi = (self.b as i64) - 1; - let mut range_prod = one.clone(); - for t in lo..=hi { - let t_u64 = if t >= 0 { - t as u64 - } else { - GOLDILOCKS_P_U64 - .checked_sub((-t) as u64) - .ok_or_else(|| Error::Synthesis("PiCcsNcChunkAggSumcheckRelation: t underflow".into()))? - }; - let t_k = k_const(std_lib, layouter, t_u64, 0)?; - let term = k_sub_mod_var(std_lib, layouter, &y_eval, &t_k)?; - range_prod = k_mul_mod_var(std_lib, layouter, &range_prod, &term, K_DELTA_U64)?; - } + // y_eval = . + let mut eval_vec = y_zcol; + for a in alpha_prime { + let next_len = eval_vec.len() / 2; + let mut next: Vec = Vec::with_capacity(next_len); + for j in 0..next_len { + let v0 = &eval_vec[2 * j]; + let v1 = &eval_vec[2 * j + 1]; + next.push(k_mle_fold_step(std_lib, layouter, v0, v1, a, K_DELTA_U64)?); + } + eval_vec = next; + } + if eval_vec.len() != 1 { + return Err(Error::Synthesis(format!( + "PiCcsNcChunkAggSumcheckRelation: eval_vec len {} != 1 (ell_d={})", + eval_vec.len(), + self.ell_d + ))); + } + let y_eval = eval_vec.first().expect("len checked").clone(); - let weighted = k_mul_mod_var(std_lib, layouter, &g, &range_prod, K_DELTA_U64)?; - weighted_terms.push(weighted); + // range_product(y_eval) = ∏_{t=-(b-1)}^{b-1} (y_eval - t) + let lo = -((self.b as i64) - 1); + let hi = (self.b as i64) - 1; + let mut range_prod = one.clone(); + for t in lo..=hi { + let t_u64 = if t >= 0 { + t as u64 + } else { + GOLDILOCKS_P_U64 + .checked_sub((-t) as u64) + .ok_or_else(|| Error::Synthesis("PiCcsNcChunkAggSumcheckRelation: t underflow".into()))? + }; + let t_k = k_const(std_lib, layouter, t_u64, 0)?; + let term = k_sub_mod_var(std_lib, layouter, &y_eval, &t_k)?; + range_prod = k_mul_mod_var(std_lib, layouter, &range_prod, &term, K_DELTA_U64)?; + } - if out_idx + 1 < self.count { - g = k_mul_mod_var(std_lib, layouter, &g, &gamma, K_DELTA_U64)?; + let weighted = k_mul_mod_var(std_lib, layouter, &g, &range_prod, K_DELTA_U64)?; + weighted_terms.push(weighted); + + if out_idx + 1 < self.count { + g = k_mul_mod_var(std_lib, layouter, &g, &gamma, K_DELTA_U64)?; + } } - } - let acc = k_sum_mod_var(std_lib, layouter, &weighted_terms)?; - assert_k_eq(std_lib, layouter, &acc, &chunk_sums[self.chunk_index])?; + let acc = k_sum_mod_var(std_lib, layouter, &weighted_terms)?; + assert_k_eq(std_lib, layouter, &acc, &chunk_sums[self.chunk_index])?; + } // Aggregate: final_sum_nc == eq((α',s'),(β_a,β_m)) * Σ chunk_sums. let eq_a = k_eq_points(std_lib, layouter, alpha_prime, &beta_a)?; @@ -2736,14 +2714,11 @@ impl Relation for PiCcsNcChunkAggregateRelation { } fn write_relation(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = bincode::serialize(self).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - writer.write_all(&bytes) + write_relation_len_prefixed(writer, self) } fn read_relation(reader: &mut R) -> std::io::Result { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - bincode::deserialize(&bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + read_relation_len_prefixed(reader) } fn circuit( diff --git a/crates/neo-midnight-bridge/testdata/kzg_params/.gitkeep b/crates/neo-midnight-bridge/testdata/kzg_params/.gitkeep new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/crates/neo-midnight-bridge/testdata/kzg_params/.gitkeep @@ -0,0 +1 @@ + diff --git a/crates/neo-midnight-bridge/testdata/plonk_keys/.gitkeep b/crates/neo-midnight-bridge/testdata/plonk_keys/.gitkeep new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/crates/neo-midnight-bridge/testdata/plonk_keys/.gitkeep @@ -0,0 +1 @@ + diff --git a/crates/neo-midnight-bridge/tests/common/mod.rs b/crates/neo-midnight-bridge/tests/common/mod.rs index c843809d..67cf12db 100644 --- a/crates/neo-midnight-bridge/tests/common/mod.rs +++ b/crates/neo-midnight-bridge/tests/common/mod.rs @@ -6,6 +6,7 @@ use blake2b_simd::State as TranscriptHash; use midnight_curves::Bls12; use midnight_proofs::dev::cost_model::circuit_model; use midnight_proofs::poly::kzg::params::ParamsKZG; +use midnight_proofs::utils::SerdeFormat; use midnight_zk_stdlib::Relation; use neo_math::{KExtensions, D, F, K}; use neo_midnight_bridge::k_field::KRepr; @@ -25,11 +26,48 @@ use std::collections::BTreeMap; use std::fs; use std::path::PathBuf; -pub const MIDNIGHT_MAX_K: u32 = 14; - const PARAMS_DIGEST_DOMAIN: &[u8] = b"neo/midnight-bridge/params-digest/v1"; const ACC_DIGEST_DOMAIN: &[u8] = b"neo/midnight-bridge/acc-digest/v1"; +fn kzg_params_testdata_path(k: u32) -> PathBuf { + // To pre-populate this folder with Midnight-provided ParamsKZG files: + // + // BASE_URL="https://midnight-s3-fileshare-dev-eu-west-1.s3.eu-west-1.amazonaws.com" + // OUT_DIR="crates/neo-midnight-bridge/testdata/kzg_params" + // mkdir -p "$OUT_DIR" + // curl -L --fail -o "$OUT_DIR/bls_midnight_2p18" "$BASE_URL/bls_midnight_2p18" + // + // Then tests will use the downloaded file instead of generating a deterministic `unsafe_setup()`. + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + manifest_dir + .join("testdata") + .join("kzg_params") + .join(format!("bls_midnight_2p{k}")) +} + +fn kzg_params_unsafe_setup_deterministic(k: u32) -> ParamsKZG { + let mut seed = [0u8; 32]; + seed[0] = 0xA5; + seed[1] = (k & 0xFF) as u8; + seed[2] = ((k >> 8) & 0xFF) as u8; + seed[3] = ((k >> 16) & 0xFF) as u8; + seed[4] = ((k >> 24) & 0xFF) as u8; + ParamsKZG::unsafe_setup(k, ChaCha20Rng::from_seed(seed)) +} + +fn try_read_kzg_params_from_testdata(k: u32) -> Option> { + let path = kzg_params_testdata_path(k); + let mut f = fs::File::open(&path).ok()?; + Some( + ParamsKZG::::read_custom(&mut f, SerdeFormat::RawBytes) + .unwrap_or_else(|e| panic!("read ParamsKZG from {path:?}: {e}")), + ) +} + +pub fn test_kzg_params(k: u32) -> ParamsKZG { + try_read_kzg_params_from_testdata(k).unwrap_or_else(|| kzg_params_unsafe_setup_deterministic(k)) +} + fn params_digest32(params: &neo_params::NeoParams) -> [u8; 32] { let bytes = bincode::serialize(params).expect("NeoParams should be serializable"); blake2b_256_domain(PARAMS_DIGEST_DOMAIN, &bytes) @@ -66,49 +104,14 @@ struct KzgParamsCache { impl KzgParamsCache { fn get(&mut self, k: u32) -> &ParamsKZG { - self.by_k.entry(k).or_insert_with(|| { - let mut seed = [0u8; 32]; - seed[0] = 0xA5; - seed[1] = (k & 0xFF) as u8; - seed[2] = ((k >> 8) & 0xFF) as u8; - seed[3] = ((k >> 16) & 0xFF) as u8; - seed[4] = ((k >> 24) & 0xFF) as u8; - ParamsKZG::unsafe_setup(k, ChaCha20Rng::from_seed(seed)) - }) + self.by_k.entry(k).or_insert_with(|| test_kzg_params(k)) } } -/// Find the maximum `count` in `[1..=max_count]` that fits `min_k <= MIDNIGHT_MAX_K`. -/// -/// Assumes `min_k` is monotone non-decreasing in `count` (true for these relations because -/// they scale linearly in `count`, so row count increases with `count`). -fn choose_max_count_under_k( - max_count: usize, - label: &str, - mut model_for_count: impl FnMut(usize) -> (u32, usize), -) -> usize { - let mut lo = 1usize; - let mut hi = max_count; - let mut best = 1usize; - let mut best_k = u32::MAX; - let mut best_rows = 0usize; - - while lo <= hi { - let mid = (lo + hi) / 2; - let (k, rows) = model_for_count(mid); - println!("{label} trial: count={mid} min_k={k} rows={rows}"); - if k <= MIDNIGHT_MAX_K { - best = mid; - best_k = k; - best_rows = rows; - lo = mid + 1; - } else { - hi = mid - 1; - } - } - - println!("Choosing count={best} (min_k={best_k} rows={best_rows}) for {label}"); - best +fn choose_max_count(max_count: usize, label: &str, mut model_for_count: impl FnMut(usize) -> (u32, usize)) -> usize { + let (k, rows) = model_for_count(max_count); + println!("{label}: count={max_count} min_k={k} rows={rows}"); + max_count } fn k_to_repr(k: &neo_math::K) -> KRepr { @@ -259,8 +262,8 @@ pub fn prove_step1_nc_bundle_poseidon2_batch_40() { let (s_col_prime, alpha_prime_nc) = pi.sumcheck_challenges_nc.split_at(ell_m); let gamma = pi.challenges_public.gamma; - // Choose the largest chunk size that fits the max-k cap (binary search). - let chunk_size = choose_max_count_under_k(k_total, "PiCcsNcChunkRelation", |count| { + // Choose the largest chunk size (one proof per step if possible). + let chunk_size = choose_max_count(k_total, "PiCcsNcChunkRelation", |count| { let rel_try = PiCcsNcChunkRelation { ell_d, b: params_b, @@ -361,10 +364,6 @@ pub fn prove_step1_nc_bundle_poseidon2_batch_40() { model.k, model.rows ); - assert!( - model.k <= MIDNIGHT_MAX_K, - "expected k<=MIDNIGHT_MAX_K for chunk+agg+sumcheck" - ); total_statement_bytes += ::format_instance(&inst_agg) .expect("format_instance") @@ -418,7 +417,6 @@ pub fn prove_step1_nc_bundle_poseidon2_batch_40() { model.k, model.rows ); - assert!(model.k <= MIDNIGHT_MAX_K, "expected k<=MIDNIGHT_MAX_K per chunk"); let params = params_cache.get(model.k); let vk = midnight_zk_stdlib::setup_vk(params, &rel); @@ -530,8 +528,8 @@ pub fn prove_step1_full_bundle_poseidon2_batch_40() { let (_s_col_prime, alpha_prime_nc) = pi.sumcheck_challenges_nc.split_at(ell_m); let gamma = pi.challenges_public.gamma; - // Choose the largest chunk size that fits the max-k cap (binary search). - let chunk_size = choose_max_count_under_k(k_total, "PiCcsNcChunkRelation", |count| { + // Choose the largest chunk size (one proof per step if possible). + let chunk_size = choose_max_count(k_total, "PiCcsNcChunkRelation", |count| { let rel_try = PiCcsNcChunkRelation { ell_d, b: params_b, @@ -630,10 +628,6 @@ pub fn prove_step1_full_bundle_poseidon2_batch_40() { model.k, model.rows ); - assert!( - model.k <= MIDNIGHT_MAX_K, - "expected k<=MIDNIGHT_MAX_K for chunk+agg+sumcheck" - ); nc_statement_bytes += ::format_instance(&inst_nc_agg) .expect("format_instance") @@ -692,7 +686,6 @@ pub fn prove_step1_full_bundle_poseidon2_batch_40() { model.k, model.rows ); - assert!(model.k <= MIDNIGHT_MAX_K, "expected k<=MIDNIGHT_MAX_K per chunk"); let params = params_cache.get(model.k); let vk = midnight_zk_stdlib::setup_vk(params, &rel); @@ -793,9 +786,9 @@ pub fn prove_step1_full_bundle_poseidon2_batch_40() { println!("FE y entries: total={total_y_entries} nonzero_c1={nonzero_c1} (c1==0 means base-field digit)"); assert_eq!(y_rows_flat.len(), (k_total - 1) * t_fe); - // Choose the largest FE chunk size (in flattened (out_idx,j) pairs) that fits Midnight's cap. + // Choose the largest FE chunk size (in flattened (out_idx,j) pairs). let total_pairs = y_rows_flat.len(); - let fe_chunk_size = choose_max_count_under_k(total_pairs, "PiCcsFeChunkRelation", |count| { + let fe_chunk_size = choose_max_count(total_pairs, "PiCcsFeChunkRelation", |count| { let rel_try = PiCcsFeChunkRelation { ell_d, k_total, @@ -867,63 +860,56 @@ pub fn prove_step1_full_bundle_poseidon2_batch_40() { model.k, model.rows ); - if model.k <= MIDNIGHT_MAX_K { - let instance = PiCcsFeChunkAggSumcheckInstance { - bundle_digest, - sumcheck_challenges: pi.sumcheck_challenges.iter().map(k_to_repr).collect(), - gamma: gamma_repr_fe, - alpha: pi.challenges_public.alpha.iter().map(k_to_repr).collect(), - beta_a: pi.challenges_public.beta_a.iter().map(k_to_repr).collect(), - beta_r: pi.challenges_public.beta_r.iter().map(k_to_repr).collect(), - chunk_sums: fe_chunk_instances.clone(), - initial_sum: initial_sum_fe, - final_sum: final_sum_fe, - }; - let witness = PiCcsFeChunkAggSumcheckWitness { - rounds: pi - .sumcheck_rounds - .iter() - .map(|r| r.iter().map(k_to_repr).collect()) - .collect(), - me_inputs_r: me_inputs_r.iter().map(k_to_repr).collect(), - y_scalars_0: out0.y_scalars.iter().map(k_to_repr).collect(), - y_rows: y_rows_flat - .iter() - .map(|row| row.iter().map(k_to_repr).collect::>()) - .collect(), - }; + let instance = PiCcsFeChunkAggSumcheckInstance { + bundle_digest, + sumcheck_challenges: pi.sumcheck_challenges.iter().map(k_to_repr).collect(), + gamma: gamma_repr_fe, + alpha: pi.challenges_public.alpha.iter().map(k_to_repr).collect(), + beta_a: pi.challenges_public.beta_a.iter().map(k_to_repr).collect(), + beta_r: pi.challenges_public.beta_r.iter().map(k_to_repr).collect(), + chunk_sums: fe_chunk_instances.clone(), + initial_sum: initial_sum_fe, + final_sum: final_sum_fe, + }; + let witness = PiCcsFeChunkAggSumcheckWitness { + rounds: pi + .sumcheck_rounds + .iter() + .map(|r| r.iter().map(k_to_repr).collect()) + .collect(), + me_inputs_r: me_inputs_r.iter().map(k_to_repr).collect(), + y_scalars_0: out0.y_scalars.iter().map(k_to_repr).collect(), + y_rows: y_rows_flat + .iter() + .map(|row| row.iter().map(k_to_repr).collect::>()) + .collect(), + }; - fe_statement_bytes += ::format_instance(&instance) - .expect("format_instance") - .len() - * 32; + fe_statement_bytes += ::format_instance(&instance) + .expect("format_instance") + .len() + * 32; - let params = params_cache.get(model.k); - let vk = midnight_zk_stdlib::setup_vk(params, &rel); - let pk = midnight_zk_stdlib::setup_pk(&rel, &vk); - let proof = midnight_zk_stdlib::prove::<_, TranscriptHash>( - params, - &pk, - &rel, - &instance, - witness, - ChaCha20Rng::from_seed([141u8; 32]), - ) - .expect("prove fe chunk+agg+sumcheck"); - fe_bundle_bytes += proof.len(); + let params = params_cache.get(model.k); + let vk = midnight_zk_stdlib::setup_vk(params, &rel); + let pk = midnight_zk_stdlib::setup_pk(&rel, &vk); + let proof = midnight_zk_stdlib::prove::<_, TranscriptHash>( + params, + &pk, + &rel, + &instance, + witness, + ChaCha20Rng::from_seed([141u8; 32]), + ) + .expect("prove fe chunk+agg+sumcheck"); + fe_bundle_bytes += proof.len(); - let params_v = params.verifier_params(); - midnight_zk_stdlib::verify::( - ¶ms_v, &vk, &instance, None, &proof, - ) - .expect("verify fe chunk+agg+sumcheck"); - did_one_shot_fe = true; - } else { - println!( - "FE one-shot requires min_k={} > MIDNIGHT_MAX_K={MIDNIGHT_MAX_K}; falling back to chunk + sumcheck+agg proofs.", - model.k - ); - } + let params_v = params.verifier_params(); + midnight_zk_stdlib::verify::( + ¶ms_v, &vk, &instance, None, &proof, + ) + .expect("verify fe chunk+agg+sumcheck"); + did_one_shot_fe = true; } if !did_one_shot_fe { @@ -959,7 +945,6 @@ pub fn prove_step1_full_bundle_poseidon2_batch_40() { model.k, model.rows ); - assert!(model.k <= MIDNIGHT_MAX_K, "expected k<=MIDNIGHT_MAX_K per FE chunk"); let params = params_cache.get(model.k); let vk = midnight_zk_stdlib::setup_vk(params, &rel); @@ -1025,10 +1010,6 @@ pub fn prove_step1_full_bundle_poseidon2_batch_40() { "FE Sumcheck+Agg: n_chunks={n_chunks_fe} min_k={} rows={}", model_fe_sc_agg.k, model_fe_sc_agg.rows ); - assert!( - model_fe_sc_agg.k <= MIDNIGHT_MAX_K, - "expected k<=MIDNIGHT_MAX_K for FE sumcheck+agg" - ); let params_fe_sc_agg = params_cache.get(model_fe_sc_agg.k); let vk_fe_sc_agg = midnight_zk_stdlib::setup_vk(params_fe_sc_agg, &rel_fe_sc_agg); diff --git a/crates/neo-midnight-bridge/tests/pi_ccs_sumcheck_nc_poseidon2_batch_40.rs b/crates/neo-midnight-bridge/tests/pi_ccs_sumcheck_nc_poseidon2_batch_40.rs index 7838db15..86a010e6 100644 --- a/crates/neo-midnight-bridge/tests/pi_ccs_sumcheck_nc_poseidon2_batch_40.rs +++ b/crates/neo-midnight-bridge/tests/pi_ccs_sumcheck_nc_poseidon2_batch_40.rs @@ -1,7 +1,7 @@ +mod common; + use blake2b_simd::State as TranscriptHash; -use midnight_curves::Bls12; use midnight_proofs::dev::cost_model::circuit_model; -use midnight_proofs::poly::kzg::params::ParamsKZG; use neo_math::KExtensions; use neo_midnight_bridge::k_field::KRepr; use neo_midnight_bridge::relations::{PiCcsSumcheckInstance, PiCcsSumcheckNcRelation, PiCcsSumcheckWitness}; @@ -99,10 +99,7 @@ fn plonk_kzg_pi_ccs_sumcheck_nc_poseidon2_batch_40_roundtrip() { model.fixed_columns, model.lookups ); - assert!(k <= 14, "expected Midnight k<=14 cap"); - - let rng = ChaCha20Rng::from_seed([31u8; 32]); - let params: ParamsKZG = ParamsKZG::unsafe_setup(k, rng); + let params = common::test_kzg_params(k); let vk = midnight_zk_stdlib::setup_vk(¶ms, &rel); let pk = midnight_zk_stdlib::setup_pk(&rel, &vk); diff --git a/crates/neo-midnight-bridge/tests/pi_ccs_terminal_fe_k1_poseidon2_batch_40.rs b/crates/neo-midnight-bridge/tests/pi_ccs_terminal_fe_k1_poseidon2_batch_40.rs index e134150c..f49ea70a 100644 --- a/crates/neo-midnight-bridge/tests/pi_ccs_terminal_fe_k1_poseidon2_batch_40.rs +++ b/crates/neo-midnight-bridge/tests/pi_ccs_terminal_fe_k1_poseidon2_batch_40.rs @@ -1,7 +1,7 @@ +mod common; + use blake2b_simd::State as TranscriptHash; -use midnight_curves::Bls12; use midnight_proofs::dev::cost_model::circuit_model; -use midnight_proofs::poly::kzg::params::ParamsKZG; use neo_math::{KExtensions, D}; use neo_midnight_bridge::k_field::KRepr; use neo_midnight_bridge::relations::{ @@ -122,10 +122,7 @@ fn plonk_kzg_pi_ccs_terminal_fe_k1_poseidon2_batch_40_roundtrip() { model.fixed_columns, model.lookups ); - assert!(k <= 14, "expected Midnight k<=14 cap"); - - let rng = ChaCha20Rng::from_seed([41u8; 32]); - let params: ParamsKZG = ParamsKZG::unsafe_setup(k, rng); + let params = common::test_kzg_params(k); let vk = midnight_zk_stdlib::setup_vk(¶ms, &rel); let pk = midnight_zk_stdlib::setup_pk(&rel, &vk); diff --git a/crates/neo-midnight-bridge/tests/pi_ccs_terminal_nc_k1_poseidon2_batch_40.rs b/crates/neo-midnight-bridge/tests/pi_ccs_terminal_nc_k1_poseidon2_batch_40.rs index d6fa7683..4736866e 100644 --- a/crates/neo-midnight-bridge/tests/pi_ccs_terminal_nc_k1_poseidon2_batch_40.rs +++ b/crates/neo-midnight-bridge/tests/pi_ccs_terminal_nc_k1_poseidon2_batch_40.rs @@ -1,7 +1,7 @@ +mod common; + use blake2b_simd::State as TranscriptHash; -use midnight_curves::Bls12; use midnight_proofs::dev::cost_model::circuit_model; -use midnight_proofs::poly::kzg::params::ParamsKZG; use neo_math::{KExtensions, D}; use neo_midnight_bridge::k_field::KRepr; use neo_midnight_bridge::relations::{PiCcsNcTerminalK1Instance, PiCcsNcTerminalK1Relation, PiCcsNcTerminalK1Witness}; @@ -110,10 +110,7 @@ fn plonk_kzg_pi_ccs_terminal_nc_k1_poseidon2_batch_40_roundtrip() { model.fixed_columns, model.lookups ); - assert!(k <= 14, "expected Midnight k<=14 cap"); - - let rng = ChaCha20Rng::from_seed([51u8; 32]); - let params: ParamsKZG = ParamsKZG::unsafe_setup(k, rng); + let params = common::test_kzg_params(k); let vk = midnight_zk_stdlib::setup_vk(¶ms, &rel); let pk = midnight_zk_stdlib::setup_pk(&rel, &vk);