From db2458b281eb6c2da85676af128d13590f04fdc2 Mon Sep 17 00:00:00 2001 From: Toufeeq Pasha Date: Tue, 7 Oct 2025 11:41:27 +0530 Subject: [PATCH 1/3] add multiproof benchmarks --- Cargo.lock | 84 +++++++++- kate/Cargo.toml | 5 + kate/benches/multiproof.rs | 310 +++++++++++++++++++++++++++++++++++++ kate/src/com.rs | 8 +- 4 files changed, 401 insertions(+), 6 deletions(-) create mode 100644 kate/benches/multiproof.rs diff --git a/Cargo.lock b/Cargo.lock index 820833d..88c071d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -115,6 +115,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + [[package]] name = "anyhow" version = "1.0.79" @@ -870,11 +876,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "bitflags 1.3.2", - "clap_lex", + "clap_lex 0.2.4", "indexmap 1.9.3", "textwrap", ] +[[package]] +name = "clap" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +dependencies = [ + "anstyle", + "clap_lex 0.7.5", + "terminal_size", +] + [[package]] name = "clap_lex" version = "0.2.4" @@ -884,6 +910,12 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + [[package]] name = "common" version = "0.1.0" @@ -905,6 +937,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101" +[[package]] +name = "condtype" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" + [[package]] name = "const-oid" version = "0.9.6" @@ -995,7 +1033,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap", + "clap 3.2.25", "criterion-plot", "itertools", "lazy_static", @@ -1208,6 +1246,31 @@ dependencies = [ "subtle", ] +[[package]] +name = "divan" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a405457ec78b8fe08b0e32b4a3570ab5dff6dd16eb9e76a5ee0a9d9cbd898933" +dependencies = [ + "cfg-if", + "clap 4.5.18", + "condtype", + "divan-macros", + "libc", + "regex-lite", +] + +[[package]] +name = "divan-macros" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9556bc800956545d6420a640173e5ba7dfa82f38d3ea5a167eb555bc69ac3323" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "dleq_vrf" version = "0.0.2" @@ -2069,6 +2132,7 @@ dependencies = [ "avail-core", "criterion", "derive_more", + "divan", "hex", "hex-literal", "kate-recovery", @@ -2905,6 +2969,12 @@ dependencies = [ "regex-syntax 0.8.2", ] +[[package]] +name = "regex-lite" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943f41321c63ef1c92fd763bfe054d2668f7f225a5c29f0105903dc2fc04ba30" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -3891,6 +3961,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" +dependencies = [ + "rustix 0.38.31", + "windows-sys 0.48.0", +] + [[package]] name = "test-case" version = "1.2.3" diff --git a/kate/Cargo.toml b/kate/Cargo.toml index 0349678..5fcea08 100644 --- a/kate/Cargo.toml +++ b/kate/Cargo.toml @@ -42,6 +42,7 @@ criterion.workspace = true proptest.workspace = true serde_json.workspace = true test-case.workspace = true +divan = { version = "0.1" } [features] default = [ "std" ] @@ -71,3 +72,7 @@ maximum-block-size = [] [[bench]] name = "reconstruct" harness = false + +[[bench]] +name = "multiproof" +harness = false \ No newline at end of file diff --git a/kate/benches/multiproof.rs b/kate/benches/multiproof.rs new file mode 100644 index 0000000..26d6b59 --- /dev/null +++ b/kate/benches/multiproof.rs @@ -0,0 +1,310 @@ +use avail_core::{BlockLengthColumns, BlockLengthRows}; +use divan::Bencher; +use kate::com::Cell; +use kate::{couscous, gridgen::core::*, Seed}; +use kate_recovery::commons::ArkPublicParams; +use kate_recovery::matrix::Dimensions; +use kate_recovery::proof::domain_points; +use poly_multiproof::merlin::Transcript; +use poly_multiproof::traits::PolyMultiProofNoPrecomp; +use rand::Rng; +use std::sync::atomic::{AtomicUsize, Ordering}; + +fn main() { + divan::main(); +} + +// #[divan::bench(max_time = 5)] +// fn build_commitments_from_data(bencher: Bencher) { +// let pp = couscous::multiproof_params(); +// let mut rng = rand::thread_rng(); +// let blob_m: Vec = (0..32_505_656u32).map(|_| rng.gen()).collect(); + +// bencher.bench(|| { +// let grid = EvaluationGrid::from_data(&blob_m, 1024, 1024, 4096, Seed::default()) +// .expect("failed to build evaluation grid"); + +// let poly_grid = grid +// .make_polynomial_grid() +// .expect("failed to build polynomial grid"); + +// let _commitments = poly_grid +// .commitments(&pp) +// .expect("failed to compute commitments"); +// }); +// } + +fn random_cell(rng: &mut impl Rng, dims: Dimensions) -> Cell { + let row = rng.gen_range(0..dims.rows().get()); + let col = rng.gen_range(0..dims.cols().get()); + Cell::new(BlockLengthRows(row as u32), BlockLengthColumns(col as u32)) +} + +fn run_proof_generate(pp: &ArkPublicParams, blob_m: &[u8], target_dims: Dimensions) { + let grid = EvaluationGrid::from_data(blob_m, 1024, 1024, 1024, Seed::default()) + .expect("failed to build evaluation grid"); + + let poly_grid = grid + .make_polynomial_grid() + .expect("failed to build polynomial grid"); + + let mut rng = rand::thread_rng(); + let cell = random_cell(&mut rng, target_dims); + + let _multiproof = poly_grid + .multiproof(pp, &cell, &grid, target_dims) + .expect("failed to build multiproof"); +} + +fn generate_multiproof_and_commitments( + pp: &ArkPublicParams, + blob_m: &[u8], + target_dims: Dimensions, + cells: Vec, +) -> ( + Vec>, + Vec, +) { + let grid = EvaluationGrid::from_data(blob_m, 1024, 1024, 1024, Seed::default()) + .expect("failed to build evaluation grid"); + + let poly_grid = grid + .make_polynomial_grid() + .expect("failed to build polynomial grid"); + let commitments = poly_grid + .commitments(pp) + .expect("failed to compute commitments"); + + let mut multiproofs = Vec::with_capacity(cells.len()); + for cell in cells { + let multiproof = poly_grid + .multiproof(pp, &cell, &grid, target_dims) + .expect("failed to build multiproof"); + multiproofs.push(multiproof); + } + (multiproofs, commitments) +} + +fn blob() -> Vec { + let mut rng = rand::thread_rng(); + (0..32_505_656u32).map(|_| rng.gen()).collect() +} + +// ---------------- Proof Generation BENCHES ---------------- // + +#[divan::bench(max_time = 10)] +fn multiproof_generate_1kib_subgrid_1024_32(bencher: Bencher) { + let pp = couscous::multiproof_params(); + let blob_m = blob(); + let target_dims = Dimensions::new_from(1024, 32).unwrap(); + + bencher.bench(|| run_proof_generate(&pp, &blob_m, target_dims)); +} + +#[divan::bench(max_time = 10)] +fn multiproof_generate_1kib_subgrid_32_1024(bencher: Bencher) { + let pp = couscous::multiproof_params(); + let blob_m = blob(); + let target_dims = Dimensions::new_from(32, 1024).unwrap(); + + bencher.bench(|| run_proof_generate(&pp, &blob_m, target_dims)); +} + +#[divan::bench(max_time = 10)] +fn multiproof_generate_8kib_subgrid_1024_4(bencher: Bencher) { + let pp = couscous::multiproof_params(); + let blob_m = blob(); + let target_dims = Dimensions::new_from(1024, 4).unwrap(); + + bencher.bench(|| run_proof_generate(&pp, &blob_m, target_dims)); +} + +#[divan::bench(max_time = 10)] +fn multiproof_generate_8kib_subgrid_4_1024(bencher: Bencher) { + let pp = couscous::multiproof_params(); + let blob_m = blob(); + let target_dims = Dimensions::new_from(4, 1024).unwrap(); + + bencher.bench(|| run_proof_generate(&pp, &blob_m, target_dims)); +} + +#[divan::bench(max_time = 10)] +fn multiproof_generate_32kib_subgrid_1_1024(bencher: Bencher) { + let pp = couscous::multiproof_params(); + let blob_m = blob(); + let target_dims = Dimensions::new_from(1, 1024).unwrap(); + + bencher.bench(|| run_proof_generate(&pp, &blob_m, target_dims)); +} + +#[divan::bench(max_time = 10)] +fn multiproof_generate_32kib_subgrid_1024_1(bencher: Bencher) { + let pp = couscous::multiproof_params(); + let blob_m = blob(); + let target_dims = Dimensions::new_from(1024, 1).unwrap(); + + bencher.bench(|| run_proof_generate(&pp, &blob_m, target_dims)); +} + +// ---------------- Proof Verification BENCHES ---------------- // + +#[divan::bench(max_time = 10)] +fn multiproof_verify_1kib_subgrid_1024_32(bencher: Bencher) { + let pp = couscous::multiproof_params(); + let blob_m = blob(); + let target_dims = Dimensions::new_from(1024, 32).unwrap(); + + // generate random cells to prove/verify + let mut rng = rand::thread_rng(); + let cells: Vec = (0..16) + .map(|_| random_cell(&mut rng, target_dims)) + .collect(); + + // precompute multiproofs + commitments for all cells + let (multiproofs, commitments) = + generate_multiproof_and_commitments(&pp, &blob_m, target_dims, cells); + + // domain points for verification + let points = domain_points(1024).unwrap(); // adjust if your API differs + + // cycle through the precomputed multiproofs + let idx = AtomicUsize::new(0); + + bencher.bench(|| { + let i = idx.fetch_add(1, Ordering::Relaxed); + let multiproof = &multiproofs[i % multiproofs.len()]; + + let verified = PolyMultiProofNoPrecomp::verify( + &pp, + &mut Transcript::new(b"avail-mp"), + &commitments[multiproof.block.start_y..multiproof.block.end_y], + &points[multiproof.block.start_x..multiproof.block.end_x], + &multiproof.evals, + &multiproof.proof, + ) + .unwrap(); + + divan::black_box(verified); + }); +} + +#[divan::bench(max_time = 10)] +fn multiproof_verify_1kib_subgrid_32_1024(bencher: Bencher) { + let pp = couscous::multiproof_params(); + let blob_m = blob(); + let target_dims = Dimensions::new_from(32, 1024).unwrap(); + + // generate random cells to prove/verify + let mut rng = rand::thread_rng(); + let cells: Vec = (0..16) + .map(|_| random_cell(&mut rng, target_dims)) + .collect(); + + // precompute multiproofs + commitments for all cells + let (multiproofs, commitments) = + generate_multiproof_and_commitments(&pp, &blob_m, target_dims, cells); + + // domain points for verification + let points = domain_points(1024).unwrap(); // adjust if your API differs + + // cycle through the precomputed multiproofs + let idx = AtomicUsize::new(0); + + bencher.bench(|| { + let i = idx.fetch_add(1, Ordering::Relaxed); + let multiproof = &multiproofs[i % multiproofs.len()]; + + let verified = PolyMultiProofNoPrecomp::verify( + &pp, + &mut Transcript::new(b"avail-mp"), + &commitments[multiproof.block.start_y..multiproof.block.end_y], + &points[multiproof.block.start_x..multiproof.block.end_x], + &multiproof.evals, + &multiproof.proof, + ) + .unwrap(); + + divan::black_box(verified); + }); +} + +#[divan::bench(max_time = 10)] +fn multiproof_verify_32kib_subgrid_1_1024(bencher: Bencher) { + let pp = couscous::multiproof_params(); + let blob_m = blob(); + let target_dims = Dimensions::new_from(1, 1024).unwrap(); + + // generate random cells to prove/verify + let mut rng = rand::thread_rng(); + let cells: Vec = (0..16) + .map(|_| random_cell(&mut rng, target_dims)) + .collect(); + + // precompute multiproofs + commitments for all cells + let (multiproofs, commitments) = + generate_multiproof_and_commitments(&pp, &blob_m, target_dims, cells); + + // domain points for verification + let points = domain_points(1024).unwrap(); // adjust if your API differs + + // cycle through the precomputed multiproofs + let idx = AtomicUsize::new(0); + + bencher.bench(|| { + let i = idx.fetch_add(1, Ordering::Relaxed); + let multiproof = &multiproofs[i % multiproofs.len()]; + + let verified = PolyMultiProofNoPrecomp::verify( + &pp, + &mut Transcript::new(b"avail-mp"), + &commitments[multiproof.block.start_y..multiproof.block.end_y], + &points[multiproof.block.start_x..multiproof.block.end_x], + &multiproof.evals, + &multiproof.proof, + ) + .unwrap(); + + divan::black_box(verified); + }); +} + +#[divan::bench(max_time = 10)] +// This benchmark requires larger g2's (1025) than currently available 513 in the pp +// fn multiproof_verify_32kib_subgrid_1024_1(bencher: Bencher) { +// let pp = couscous::multiproof_params(); +// let blob_m = blob(); +// let target_dims = Dimensions::new_from(1024, 1).unwrap(); + +// // generate random cells to prove/verify +// let mut rng = rand::thread_rng(); +// let cells: Vec = (0..16) +// .map(|_| random_cell(&mut rng, target_dims)) +// .collect(); + +// // precompute multiproofs + commitments for all cells +// let (multiproofs, commitments) = +// generate_multiproof_and_commitments(&pp, &blob_m, target_dims, cells); + +// // domain points for verification +// let points = domain_points(1024).unwrap(); // adjust if your API differs + +// // cycle through the precomputed multiproofs +// let idx = AtomicUsize::new(0); + +// bencher.bench(|| { +// let i = idx.fetch_add(1, Ordering::Relaxed); +// let multiproof = &multiproofs[i % multiproofs.len()]; + +// let verified = PolyMultiProofNoPrecomp::verify( +// &pp, +// &mut Transcript::new(b"avail-mp"), +// &commitments[multiproof.block.start_y..multiproof.block.end_y], +// &points[multiproof.block.start_x..multiproof.block.end_x], +// &multiproof.evals, +// &multiproof.proof, +// ) +// .unwrap(); + +// divan::black_box(verified); +// }); +// } diff --git a/kate/src/com.rs b/kate/src/com.rs index 052504b..b8ec182 100644 --- a/kate/src/com.rs +++ b/kate/src/com.rs @@ -1957,9 +1957,9 @@ Let's see how this gets encoded and then reconstructed by sampling only some dat #[test] fn test_multiproof_verification_from_data() { - let rows: u16 = 1; - let cols: u16 = 16; - let target_dims = Dimensions::new_from(1, 8).unwrap(); + let rows: u16 = 8; + let cols: u16 = 8; + let target_dims = Dimensions::new_from(4, 4).unwrap(); // Compute transaction size let tx_size: u32 = rows as u32 * cols as u32 * 31; @@ -2000,7 +2000,7 @@ Let's see how this gets encoded and then reconstructed by sampling only some dat let verified = PolyMultiProofNoPrecomp::verify( &pp, &mut Transcript::new(b"avail-mp"), - &commitments, + &commitments[block.start_y..block.end_y], &points[block.start_x..block.end_x], &evals, &proof, From 74a0b0643cc3e08a28b1ec310061aea9ac73eaaf Mon Sep 17 00:00:00 2001 From: Toufeeq Pasha Date: Wed, 8 Oct 2025 09:01:08 +0530 Subject: [PATCH 2/3] fix error messages in const generic tests with latest compiler --- core/tests/const_generics/usize_even_fail.stderr | 10 +++++++--- .../const_generics/usize_greater_or_eq_fail.stderr | 10 +++++++--- core/tests/const_generics/usize_non_zero_fail.stderr | 10 +++++++--- .../const_generics/usize_safe_cast_to_u32_fail.stderr | 10 +++++++--- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/core/tests/const_generics/usize_even_fail.stderr b/core/tests/const_generics/usize_even_fail.stderr index e09c866..758e9f6 100644 --- a/core/tests/const_generics/usize_even_fail.stderr +++ b/core/tests/const_generics/usize_even_fail.stderr @@ -1,10 +1,14 @@ -error[E0080]: evaluation of `avail_core::const_generic_asserts::UsizeEven::<1>::OK` failed +error[E0080]: evaluation panicked: must be even --> src/const_generic_asserts.rs | | pub const OK: () = assert!(N % 2 == 0, "must be even"); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'must be even', $DIR/src/const_generic_asserts.rs:12:24 + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ evaluation of `avail_core::const_generic_asserts::UsizeEven::<1>::OK` failed here + +note: erroneous constant encountered + --> tests/const_generics/usize_even_fail.rs:4:11 | - = note: this error originates in the macro `$crate::panic::panic_2021` which comes from the expansion of the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info) +4 | let () = UsizeEven::::OK; + | ^^^^^^^^^^^^^^^^^^ note: the above error was encountered while instantiating `fn const_generic_even::<1>` --> tests/const_generics/usize_even_fail.rs:8:2 diff --git a/core/tests/const_generics/usize_greater_or_eq_fail.stderr b/core/tests/const_generics/usize_greater_or_eq_fail.stderr index 7c30f27..2e7912c 100644 --- a/core/tests/const_generics/usize_greater_or_eq_fail.stderr +++ b/core/tests/const_generics/usize_greater_or_eq_fail.stderr @@ -1,10 +1,14 @@ -error[E0080]: evaluation of `avail_core::const_generic_asserts::USizeGreaterOrEq::<1, 2>::OK` failed +error[E0080]: evaluation panicked: must be greater or equal --> src/const_generic_asserts.rs | | pub const OK: () = assert!(N >= M, "must be greater or equal"); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'must be greater or equal', $DIR/src/const_generic_asserts.rs:27:24 + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ evaluation of `avail_core::const_generic_asserts::USizeGreaterOrEq::<1, 2>::OK` failed here + +note: erroneous constant encountered + --> tests/const_generics/usize_greater_or_eq_fail.rs:4:11 | - = note: this error originates in the macro `$crate::panic::panic_2021` which comes from the expansion of the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info) +4 | let () = USizeGreaterOrEq::::OK; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ note: the above error was encountered while instantiating `fn const_generic_ge::<1, 2>` --> tests/const_generics/usize_greater_or_eq_fail.rs:8:2 diff --git a/core/tests/const_generics/usize_non_zero_fail.stderr b/core/tests/const_generics/usize_non_zero_fail.stderr index eb1bf79..7ea88d9 100644 --- a/core/tests/const_generics/usize_non_zero_fail.stderr +++ b/core/tests/const_generics/usize_non_zero_fail.stderr @@ -1,10 +1,14 @@ -error[E0080]: evaluation of `avail_core::const_generic_asserts::UsizeNonZero::<0>::OK` failed +error[E0080]: evaluation panicked: must be non-zero --> src/const_generic_asserts.rs | | pub const OK: () = assert!(N != 0, "must be non-zero"); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'must be non-zero', $DIR/src/const_generic_asserts.rs:6:24 + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ evaluation of `avail_core::const_generic_asserts::UsizeNonZero::<0>::OK` failed here + +note: erroneous constant encountered + --> tests/const_generics/usize_non_zero_fail.rs:4:11 | - = note: this error originates in the macro `$crate::panic::panic_2021` which comes from the expansion of the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info) +4 | let () = UsizeNonZero::::OK; + | ^^^^^^^^^^^^^^^^^^^^^ note: the above error was encountered while instantiating `fn const_generic_non_zero::<0>` --> tests/const_generics/usize_non_zero_fail.rs:8:2 diff --git a/core/tests/const_generics/usize_safe_cast_to_u32_fail.stderr b/core/tests/const_generics/usize_safe_cast_to_u32_fail.stderr index 6c22838..a4121c3 100644 --- a/core/tests/const_generics/usize_safe_cast_to_u32_fail.stderr +++ b/core/tests/const_generics/usize_safe_cast_to_u32_fail.stderr @@ -1,4 +1,4 @@ -error[E0080]: evaluation of `avail_core::const_generic_asserts::USizeSafeCastToU32::<4294967296>::OK` failed +error[E0080]: evaluation panicked: must be safe to cast to u32 --> src/const_generic_asserts.rs | | pub const OK: () = assert!( @@ -6,9 +6,13 @@ error[E0080]: evaluation of `avail_core::const_generic_asserts::USizeSafeCastToU | | size_of::() <= size_of::() || N <= u32::MAX as usize, | | "must be safe to cast to u32" | | ); - | |_____^ the evaluated program panicked at 'must be safe to cast to u32', $DIR/src/const_generic_asserts.rs:18:24 + | |_____^ evaluation of `avail_core::const_generic_asserts::USizeSafeCastToU32::<4294967296>::OK` failed here + +note: erroneous constant encountered + --> tests/const_generics/usize_safe_cast_to_u32_fail.rs:4:11 | - = note: this error originates in the macro `$crate::panic::panic_2021` which comes from the expansion of the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info) +4 | let () = USizeSafeCastToU32::::OK; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ note: the above error was encountered while instantiating `fn const_generic_safe_cast_to_u32::<4294967296>` --> tests/const_generics/usize_safe_cast_to_u32_fail.rs:8:2 From 9621a7da5762c4ecfa74d3d2d91528f4925a21ef Mon Sep 17 00:00:00 2001 From: Toufeeq Pasha Date: Wed, 8 Oct 2025 09:48:49 +0530 Subject: [PATCH 3/3] Refactor dimensions handling to use u32 instead of u16 for rows --- core/src/header/extension/mod.rs | 7 +- core/src/header/extension/v4.rs | 6 +- core/src/kate_commitment.rs | 47 +++++++++++ kate/examples/multiproof_verification.rs | 4 +- kate/recovery/src/com.rs | 2 +- kate/recovery/src/matrix.rs | 57 +++++++------ kate/src/com.rs | 102 +++++++++++++++++++++-- kate/src/gridgen/core.rs | 36 ++++---- kate/src/gridgen/tests/commitments.rs | 20 ++--- kate/src/gridgen/tests/formatting.rs | 12 +-- kate/src/gridgen/tests/mod.rs | 2 +- kate/src/gridgen/tests/reconstruction.rs | 12 +-- kate/src/lib.rs | 4 +- 13 files changed, 227 insertions(+), 84 deletions(-) diff --git a/core/src/header/extension/mod.rs b/core/src/header/extension/mod.rs index 2f2ed85..c289577 100644 --- a/core/src/header/extension/mod.rs +++ b/core/src/header/extension/mod.rs @@ -51,8 +51,11 @@ impl HeaderExtension { } } - pub fn rows(&self) -> u16 { - forward_to_version!(self, rows) + pub fn rows(&self) -> u32 { + match self { + HeaderExtension::V3(ext) => ext.rows() as u32, + HeaderExtension::V4(ext) => ext.rows(), + } } pub fn cols(&self) -> u16 { diff --git a/core/src/header/extension/v4.rs b/core/src/header/extension/v4.rs index 0890195..4ff1abc 100644 --- a/core/src/header/extension/v4.rs +++ b/core/src/header/extension/v4.rs @@ -1,4 +1,4 @@ -use crate::{data_lookup::v4::DataLookup, v3::KateCommitment}; +use crate::{data_lookup::v4::DataLookup, v4::KateCommitment}; use codec::{Decode, Encode}; use primitive_types::H256; use scale_info::TypeInfo; @@ -15,7 +15,7 @@ use sp_debug_derive::RuntimeDebug; #[cfg_attr(feature = "runtime", derive(RuntimeDebug))] pub struct HeaderExtension { pub app_lookup: DataLookup, - // Using KateCommitment from v3 since it is the same structure, although the data_root contruction has changed + // Using KateCommitment from v4 as datatype for rows is u32 pub commitment: KateCommitment, } @@ -28,7 +28,7 @@ impl HeaderExtension { &self.app_lookup } - pub fn rows(&self) -> u16 { + pub fn rows(&self) -> u32 { self.commitment.rows } diff --git a/core/src/kate_commitment.rs b/core/src/kate_commitment.rs index 077eaf7..84c4b6f 100644 --- a/core/src/kate_commitment.rs +++ b/core/src/kate_commitment.rs @@ -39,6 +39,53 @@ pub mod v3 { } } + impl fmt::Debug for KateCommitment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let commitment: &[u8] = self.commitment.as_slice(); + let data_root: &[u8] = self.data_root.as_ref(); + + f.debug_struct("KateCommitment(v4)") + .field("rows", &self.rows) + .field("cols", &self.cols) + .field("commitment", &HexDisplay(commitment)) + .field("data_root", &HexDisplay(data_root)) + .finish() + } + } +} + +pub mod v4 { + use super::*; + + /// Customized extrinsics root to save the commitment. + #[derive(PartialEq, Eq, Clone, Default, Encode, Decode, TypeInfo)] + #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] + #[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] + #[cfg_attr(feature = "serde", serde(deny_unknown_fields))] + pub struct KateCommitment { + /// Rows + #[codec(compact)] + pub rows: u32, + /// Cols + #[codec(compact)] + pub cols: u16, + /// Ark commitment. + pub commitment: Vec, + /// The merkle root of the data submitted + pub data_root: H256, + } + + impl KateCommitment { + pub fn new(rows: u32, cols: u16, data_root: H256, commitment: Vec) -> Self { + Self { + rows, + cols, + commitment, + data_root, + } + } + } + impl fmt::Debug for KateCommitment { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let commitment: &[u8] = self.commitment.as_slice(); diff --git a/kate/examples/multiproof_verification.rs b/kate/examples/multiproof_verification.rs index 5945e92..db09734 100644 --- a/kate/examples/multiproof_verification.rs +++ b/kate/examples/multiproof_verification.rs @@ -1,5 +1,4 @@ use avail_core::{AppExtrinsic, AppId, BlockLengthColumns, BlockLengthRows}; -use core::num::NonZeroU16; use hex_literal::hex; use kate::{ couscous::multiproof_params, @@ -12,6 +11,7 @@ use poly_multiproof::ark_bls12_381::Bls12_381; use poly_multiproof::method1::M1NoPrecomp; use poly_multiproof::msm::blst::BlstMSMEngine; use poly_multiproof::traits::AsBytes; +use std::num::NonZeroU32; use thiserror_no_std::Error; #[derive(Error, Debug)] @@ -46,7 +46,7 @@ fn multiproof_verification() -> Result { .collect::>(); let seed = Seed::default(); let grid = EvaluationGrid::from_extrinsics(exts, 4, 256, 256, seed)? - .extend_columns(unsafe { NonZeroU16::new_unchecked(2) })?; + .extend_columns(unsafe { NonZeroU32::new_unchecked(2) })?; // Setup, serializing as bytes let polys = grid.make_polynomial_grid()?; diff --git a/kate/recovery/src/com.rs b/kate/recovery/src/com.rs index cac4912..d888875 100644 --- a/kate/recovery/src/com.rs +++ b/kate/recovery/src/com.rs @@ -380,7 +380,7 @@ pub fn unflatten_padded_data( ) -> Result, UnflattenError> { ensure!(data.len() % CHUNK_SIZE == 0, UnflattenError::InvalidLen); - fn extract_encoded_extrinsic<'a>(range_data: &'a[u8]) -> SparseSliceRead<'a> { + fn extract_encoded_extrinsic<'a>(range_data: &'a [u8]) -> SparseSliceRead<'a> { const_assert_ne!(CHUNK_SIZE, 0); const_assert_ne!(DATA_CHUNK_SIZE, 0); diff --git a/kate/recovery/src/matrix.rs b/kate/recovery/src/matrix.rs index 536ebf2..834f63e 100644 --- a/kate/recovery/src/matrix.rs +++ b/kate/recovery/src/matrix.rs @@ -2,7 +2,7 @@ use avail_core::constants::kate::{CHUNK_SIZE, EXTENSION_FACTOR}; use core::{ convert::TryInto, fmt::{Display, Formatter, Result}, - num::NonZeroU16, + num::{NonZeroU16, NonZeroU32}, ops::{Mul, Range}, }; use derive_more::Constructor; @@ -95,21 +95,21 @@ impl RowIndex { /// /// # Example of 2x4 matrix /// -/// Data: [1,2,3,4,5,6,7,8] -/// Data rows: [1,2,3,4], [5,6,7,8] -/// Columns: [1,5], [2,6], [3,7], [4,8] -/// Extended columns (EC is erasure code): [1,EC,5,EC], [2,EC,6,EC], [3,EC,7,EC], [4,EC,8,EC] -/// Matrix representation: [1,5,2,6,3,7,4,8] +/// Data: [1,2,3,4,5,6,7,8] +/// Data rows: [1,2,3,4], [5,6,7,8] +/// Columns: [1,5], [2,6], [3,7], [4,8] +/// Extended columns (EC is erasure code): [1,EC,5,EC], [2,EC,6,EC], [3,EC,7,EC], [4,EC,8,EC] +/// Matrix representation: [1,5,2,6,3,7,4,8] /// Extended matrix representation: [1,EC,5,EC,2,EC,6,EC,3,EC,7,EC,4,EC,8,EC] #[derive(Copy, Debug, Clone, PartialEq, Eq)] pub struct Dimensions { - rows: NonZeroU16, + rows: NonZeroU32, cols: NonZeroU16, } impl From<(R, C)> for Dimensions where - R: Into, + R: Into, C: Into, { fn from(rows_cols: (R, C)) -> Self { @@ -123,7 +123,7 @@ where impl From for (R, C) where - R: From, + R: From, C: From, { fn from(d: Dimensions) -> Self { @@ -132,15 +132,15 @@ where } impl Dimensions { - pub fn new, C: TryInto>(rows: R, cols: C) -> Option { + pub fn new, C: TryInto>(rows: R, cols: C) -> Option { let rows = rows.try_into().ok()?; let cols = cols.try_into().ok()?; Some(Self { rows, cols }) } - pub fn new_from, C: TryInto>(rows: R, cols: C) -> Option { - let rows: u16 = rows.try_into().ok()?; + pub fn new_from, C: TryInto>(rows: R, cols: C) -> Option { + let rows: u32 = rows.try_into().ok()?; let cols: u16 = cols.try_into().ok()?; Self::new(rows, cols) @@ -151,16 +151,16 @@ impl Dimensions { /// /// # Safety /// Parameters `rows` and `cols` must not be zero. - pub const unsafe fn new_unchecked(rows: u16, cols: u16) -> Self { + pub const unsafe fn new_unchecked(rows: u32, cols: u16) -> Self { Self { - rows: NonZeroU16::new_unchecked(rows), + rows: NonZeroU32::new_unchecked(rows), cols: NonZeroU16::new_unchecked(cols), } } /// Returns number of rows #[inline] - pub fn rows(&self) -> NonZeroU16 { + pub fn rows(&self) -> NonZeroU32 { self.rows } @@ -170,7 +170,7 @@ impl Dimensions { /// As internal member is `NonZeroU16`, this always returns greater than zero. #[inline] pub fn height(&self) -> usize { - NonZeroU16::get(self.rows).into() + NonZeroU32::get(self.rows) as usize } /// Returns number of columns @@ -189,16 +189,16 @@ impl Dimensions { } /// Matrix size. - pub fn size + Mul>(&self) -> T { + pub fn size + From + Mul>(&self) -> T { T::from(self.rows.get()) * T::from(self.cols.get()) } pub fn divides(&self, other: &Self) -> bool { - other.cols.get() % self.cols == 0u16 && other.rows.get() % self.rows == 0u16 + other.cols.get() % self.cols == 0u16 && other.rows.get() % self.rows == 0u32 } /// Extends rows by `row_factor` and cols by `col_factor`. - pub fn extend(&self, row_factor: NonZeroU16, col_factor: NonZeroU16) -> Option { + pub fn extend(&self, row_factor: NonZeroU32, col_factor: NonZeroU16) -> Option { let rows = self.rows.checked_mul(row_factor)?; let cols = self.cols.checked_mul(col_factor)?; @@ -358,11 +358,18 @@ impl Dimensions { }) } - pub fn transpose(self) -> Self { - Self { - rows: self.cols, - cols: self.rows, - } + pub fn as_usize(&self) -> (usize, usize) { + (self.rows.get() as usize, self.cols.get() as usize) + } + + // This function has potentiial data loss if rows > u16::MAX, but its only being used in tests + pub fn transpose(self) -> Option { + let cols_u16: u16 = self.rows.get().try_into().ok()?; + let cols = NonZeroU16::new(cols_u16)?; + Some(Self { + rows: self.cols.into(), + cols, + }) } } @@ -383,7 +390,7 @@ mod tests { } #[test_case(2, 4, vec![(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3)] ; "2*4 matrix data iteration")] - fn iter_data(rows: u16, cols: u16, expected: Vec<(usize, usize)>) { + fn iter_data(rows: u32, cols: u16, expected: Vec<(usize, usize)>) { let dimensions = Dimensions::new(rows, cols).unwrap(); let cells = dimensions.iter_data().collect::>(); assert_eq!(cells, expected); diff --git a/kate/src/com.rs b/kate/src/com.rs index b8ec182..d5f6737 100644 --- a/kate/src/com.rs +++ b/kate/src/com.rs @@ -331,8 +331,8 @@ pub fn par_extend_data_matrix( let (ext_rows, _): (usize, usize) = dims .extend(ROW_EXTENSION, COL_EXTENSION) .ok_or(Error::InvalidDimensionExtension)? - .into(); - let (rows, cols) = dims.into(); + .as_usize(); + let (rows, cols) = dims.as_usize(); // simple length with mod check would work... let chunk_size = @@ -385,8 +385,8 @@ pub fn build_proof( let (ext_rows, ext_cols): (usize, usize) = dims .extend(ROW_EXTENSION, COL_EXTENSION) .ok_or(Error::InvalidDimensionExtension)? - .into(); - let (_, cols): (usize, usize) = dims.into(); + .as_usize(); + let (_, cols): (usize, usize) = dims.as_usize(); const SPROOF_SIZE: usize = PROOF_SIZE + SCALAR_SIZE; @@ -668,7 +668,6 @@ mod tests { V3DataLookup::DataLookup as DataLookupV3, }; use codec::{Compact, CompactLen, Decode}; - use core::num::NonZeroU16; use core::usize; use hex_literal::hex; use kate_recovery::proof::domain_points; @@ -1406,10 +1405,10 @@ mod tests { } println!("Commitments1 (hex): {}", hex::encode(&commitments2)); let grid1 = grid1 - .extend_columns(NonZeroU16::new(2).expect("2>0")) + .extend_columns(NonZeroU32::new(2).expect("2>0")) .unwrap(); let grid2 = grid2 - .extend_columns(NonZeroU16::new(2).expect("2>0")) + .extend_columns(NonZeroU32::new(2).expect("2>0")) .unwrap(); // merge the grids let grids = vec![grid1, grid2]; @@ -1484,6 +1483,93 @@ mod tests { } } + #[test] + fn test_large_grid() { + // 32 MB single tx + const TX_SIZE: usize = 31 * 1024 * 1024 - 200; + + let data1: Vec = [8u8; TX_SIZE].to_vec(); + let grids = vec![ + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + EvaluationGrid::from_data(&data1, 1024, 1024, 1024, Seed::default()).unwrap(), + ]; + let block_grid = EvaluationGrid::merge_with_padding(grids).unwrap(); + // println!("grid dims: {:?}", block_grid.dims()); + let poly_grid1 = block_grid + .make_polynomial_grid() + .map_err(|e| format!("Make polynomial grid failed: {e:?}")) + .unwrap(); + + let public_params = couscous::multiproof_params(); + let commitments = poly_grid1 + .extended_commitments(&public_params, 2) + .map_err(|e| format!("Grid extension failed: {e:?}")) + .unwrap(); + println!("Number of commitments: {}", commitments.len()); + } + #[test] fn test_commitments_consistency() { let tx_size: usize = 50 * 1024 * 32 - 1032; @@ -1896,7 +1982,7 @@ Let's see how this gets encoded and then reconstructed by sampling only some dat .expect("Failed to create evaluation grids"); println!("orginal grid dims: {:?}", grid.dims()); let extended_grid = grid - .extend_columns(NonZeroU16::new(2).expect("2>0")) + .extend_columns(NonZeroU32::new(2).expect("2>0")) .expect("Failed to extend columns"); println!("extended grid dims: {:?}", extended_grid.dims()); let mut app_rows: Vec<(AppId, usize)> = Vec::new(); diff --git a/kate/src/gridgen/core.rs b/kate/src/gridgen/core.rs index 4112a02..bf4bf0d 100644 --- a/kate/src/gridgen/core.rs +++ b/kate/src/gridgen/core.rs @@ -14,7 +14,7 @@ use codec::Encode; use core::{ cmp::{max, min}, iter, - num::NonZeroU16, + num::{NonZeroU16, NonZeroU32}, }; use kate_recovery::matrix::Dimensions; use nalgebra::base::DMatrix; @@ -114,7 +114,7 @@ impl EvaluationGrid { let lookup = DataLookup::from_id_and_len_iter(len_by_app)?; let grid_size = usize::try_from(lookup.len())?; let (rows, cols): (usize, usize) = - get_block_dims(grid_size, min_width, max_width, max_height)?.into(); + get_block_dims(grid_size, min_width, max_width, max_height)?.as_usize(); let mut rng = ChaChaRng::from_seed(rng_seed); // Flatten the grid @@ -153,7 +153,7 @@ impl EvaluationGrid { let grid_size = scalars.len(); let (rows, cols): (usize, usize) = - get_tx_dims(grid_size, min_width, max_width, max_height)?.into(); + get_tx_dims(grid_size, min_width, max_width, max_height)?.as_usize(); let mut rng = ChaChaRng::from_seed(rng_seed); // Flatten the grid @@ -261,8 +261,8 @@ impl EvaluationGrid { pub fn dims(&self) -> Dimensions { let (rows, cols) = self.evals.shape(); // SAFETY: We cannot construct an `EvaluationGrid` with any dimension `< 1` or `> u16::MAX` - debug_assert!(rows <= usize::from(u16::MAX) && cols <= usize::from(u16::MAX)); - unsafe { Dimensions::new_unchecked(rows as u16, cols as u16) } + debug_assert!(rows <= u32::MAX as usize && cols <= usize::from(u16::MAX)); + unsafe { Dimensions::new_unchecked(rows as u32, cols as u16) } } #[inline] @@ -283,7 +283,7 @@ impl EvaluationGrid { maybe_orig_dims: Option, ) -> Result)>>, AppRowError> { let dims = self.dims(); - let (rows, _cols): (usize, usize) = dims.into(); + let (rows, _cols): (usize, usize) = dims.as_usize(); // Ensure `origin_dims` is divisible by `dims` if some. let orig_dims = match maybe_orig_dims { @@ -298,7 +298,7 @@ impl EvaluationGrid { // Compiler checks that `Dimensions::rows()` returns a `NonZeroU16` using the expression // `NonZeroU16::get(x)` instead of `x.get()`. #[allow(clippy::arithmetic_side_effects)] - let h_mul: usize = rows / usize::from(NonZeroU16::get(orig_dims.rows())); + let h_mul: usize = rows / orig_dims.rows().get() as usize; #[allow(clippy::arithmetic_side_effects)] let row_from_lineal_index = |cols, lineal_index| { let lineal_index = @@ -329,13 +329,13 @@ impl EvaluationGrid { Ok(app_rows) } - pub fn extend_columns(&self, row_factor: NonZeroU16) -> Result { + pub fn extend_columns(&self, row_factor: NonZeroU32) -> Result { let dims = self.dims(); let (new_rows, new_cols): (usize, usize) = dims .extend(row_factor, unsafe { NonZeroU16::new_unchecked(1) }) .ok_or(Error::CellLengthExceeded)? - .into(); - let (rows, _cols): (usize, usize) = dims.into(); + .as_usize(); + let (rows, _cols): (usize, usize) = dims.as_usize(); let domain = GeneralEvaluationDomain::::new(rows).ok_or(Error::DomainSizeInvalid)?; @@ -599,14 +599,14 @@ pub fn multiproof_block( target: Dimensions, ) -> Option { let mp_grid_dims = multiproof_dims(grid, target)?; - let (g_rows, g_cols): (usize, usize) = grid.into(); + let (g_rows, g_cols): (usize, usize) = grid.as_usize(); if x >= mp_grid_dims.width() || y >= mp_grid_dims.height() { return None; } // SAFETY: Division is safe because `cols() != 0 && rows() != 0`. let block_width = g_cols / usize::from(NonZeroU16::get(mp_grid_dims.cols())); - let block_height = g_rows / usize::from(NonZeroU16::get(mp_grid_dims.rows())); + let block_height = g_rows / mp_grid_dims.rows().get() as usize; // SAFETY: values never overflow since `x` and `y` are always less than grid_dims.{width,height}(). // This is because x,y < mp_grid_dims.{width, height} and block width is the quotient of @@ -648,7 +648,7 @@ pub fn get_block_dims( .ok_or(Error::BlockTooBig)?, min_width, ); - let height = unsafe { NonZeroU16::new_unchecked(1) }; + let height = unsafe { NonZeroU32::new_unchecked(1) }; Dimensions::new_from(height, width).ok_or(Error::ZeroDimension) } else { @@ -684,7 +684,7 @@ pub fn get_tx_dims( .ok_or(Error::BlockTooBig)?, min_width, ); - let height = unsafe { NonZeroU16::new_unchecked(1) }; + let height = unsafe { NonZeroU32::new_unchecked(1) }; Dimensions::new_from(height, width).ok_or(Error::ZeroDimension) } else { @@ -756,15 +756,15 @@ mod unit_tests { #[test_case(256, 8, 32, 32 => Some((32, 8)))] #[test_case(4 , 1, 32, 32 => Some((4, 1)))] fn test_multiproof_dims( - grid_w: u16, + grid_w: u32, grid_h: u16, - target_w: u16, + target_w: u32, target_h: u16, ) -> Option<(usize, usize)> { let grid = unsafe { Dimensions::new_unchecked(grid_w, grid_h) }; let target = unsafe { Dimensions::new_unchecked(target_w, target_h) }; - multiproof_dims(grid, target).map(Into::into) + multiproof_dims(grid, target).map(|d| d.as_usize()) } use proptest::prelude::*; @@ -792,7 +792,7 @@ mod unit_tests { i.next_power_of_two() } - fn new_dim(rows: u16, cols: u16) -> Result { + fn new_dim(rows: u32, cols: u16) -> Result { Dimensions::new(rows, cols).ok_or(Error::BlockTooBig) } diff --git a/kate/src/gridgen/tests/commitments.rs b/kate/src/gridgen/tests/commitments.rs index 1d37834..dc29739 100644 --- a/kate/src/gridgen/tests/commitments.rs +++ b/kate/src/gridgen/tests/commitments.rs @@ -1,7 +1,7 @@ use super::*; use crate::{com::Cell, couscous, gridgen::core::*, Seed}; use avail_core::{AppExtrinsic, BlockLengthColumns, BlockLengthRows}; -use core::num::NonZeroU16; +use core::num::NonZeroU32; use hex_literal::hex; use kate_recovery::{ commitments::verify_equality, @@ -30,7 +30,7 @@ fn test_build_commitments_simple_commitment_check() { ) .unwrap(); let ext_evals = evals - .extend_columns(unsafe { NonZeroU16::new_unchecked(2) }) + .extend_columns(unsafe { NonZeroU32::new_unchecked(2) }) .unwrap(); let polys = ext_evals.make_polynomial_grid().unwrap(); let commits = polys @@ -63,7 +63,7 @@ fn par_build_commitments_row_wise_constant_row() { let evals = EvaluationGrid::from_extrinsics(xts, 4, 4, 4, hash).unwrap(); let evals = evals - .extend_columns(unsafe { NonZeroU16::new_unchecked(2) }) + .extend_columns(unsafe { NonZeroU32::new_unchecked(2) }) .unwrap(); let polys = evals.make_polynomial_grid().unwrap(); polys.commitments(&*PMP).unwrap(); @@ -75,8 +75,8 @@ proptest! { fn commitments_verify(ref exts in app_extrinsics_strategy()) { //let (layout, commitments, dims, matrix) = par_build_commitments(BlockLengthRows(64), BlockLengthColumns(16), 32, xts, Seed::default()).unwrap(); let grid = EvaluationGrid::from_extrinsics(exts.clone(), 4, 16, 64, Seed::default()).unwrap(); - let grid = grid.extend_columns( unsafe { NonZeroU16::new_unchecked(2)}).unwrap(); - let (g_rows, g_cols) :(u16,u16) = grid.dims().into(); + let grid = grid.extend_columns( unsafe { NonZeroU32::new_unchecked(2)}).unwrap(); + let (g_rows, g_cols) :(u32,u16) = grid.dims().into(); let orig_dims = Dimensions::new(g_rows / 2, g_cols).unwrap(); let polys = grid.make_polynomial_grid().unwrap(); let commits = polys.commitments(&*PMP) @@ -90,7 +90,7 @@ proptest! { for xt in exts.iter() { let rows = grid.app_rows(xt.app_id, Some(orig_dims)).unwrap().unwrap(); // Have to put the rows we find in this funky data structure - let mut app_rows = vec![None; g_rows.into()]; + let mut app_rows = vec![None; g_rows as usize]; for (row_i, row) in rows { app_rows[row_i] = Some(row.iter().flat_map(|s| s.to_bytes().unwrap()).collect()); } @@ -102,8 +102,8 @@ proptest! { } fn verify_commitments_missing_row(ref xts in app_extrinsics_strategy()) { - let grid = EvaluationGrid::from_extrinsics(xts.clone(), 4, 16, 64, Seed::default()).unwrap().extend_columns( unsafe { NonZeroU16::new_unchecked(2) }).unwrap(); - let (g_rows, g_cols):(u16,u16) = grid.dims().into(); + let grid = EvaluationGrid::from_extrinsics(xts.clone(), 4, 16, 64, Seed::default()).unwrap().extend_columns( unsafe { NonZeroU32::new_unchecked(2) }).unwrap(); + let (g_rows, g_cols):(u32,u16) = grid.dims().into(); let orig_dims = Dimensions::new_from(g_rows / 2, g_cols).unwrap(); let polys = grid.make_polynomial_grid().unwrap(); let commits = polys.commitments(&*PMP) @@ -116,14 +116,14 @@ proptest! { for xt in xts { let rows = grid.app_rows(xt.app_id, Some(orig_dims)).unwrap().unwrap(); - let mut row_elems = vec![None; g_rows.into()]; + let mut row_elems = vec![None; g_rows as usize]; for (i, data) in &rows { row_elems[*i] = Some(data.iter().flat_map(|s| s.to_bytes().unwrap()).collect()); } let first_index = rows.iter().map(|(i, _)| *i).min().unwrap(); row_elems.remove(first_index); - let extended_dims = orig_dims.transpose(); + let extended_dims = orig_dims.transpose().unwrap(); let (_, missing) = verify_equality(&public_params, &commits, &row_elems,&grid.lookup,extended_dims,xt.app_id).unwrap(); prop_assert!(!missing.is_empty()); } diff --git a/kate/src/gridgen/tests/formatting.rs b/kate/src/gridgen/tests/formatting.rs index ab11adc..b6308ec 100644 --- a/kate/src/gridgen/tests/formatting.rs +++ b/kate/src/gridgen/tests/formatting.rs @@ -14,7 +14,7 @@ use crate::{ gridgen::{core::EvaluationGrid, tests::sample_cells}, ArkScalar, Seed, }; -use core::num::NonZeroU16; +use core::num::NonZeroU32; #[test] fn newapi_test_flatten_block() { @@ -92,7 +92,7 @@ fn newapi_test_extend_data_matrix() { evals: DMatrix::from_row_iterator(2, 4, scalars), }; let extend = grid - .extend_columns(unsafe { NonZeroU16::new_unchecked(2) }) + .extend_columns(unsafe { NonZeroU32::new_unchecked(2) }) .unwrap(); assert_eq!(extend.evals, expected_result); @@ -100,7 +100,7 @@ fn newapi_test_extend_data_matrix() { #[test] fn test_decode_app_extrinsics() { - let app_id_1_data = br#""This is mocked test data. It will be formatted as a matrix of BLS scalar cells and then individual columns + let app_id_1_data = br#""This is mocked test data. It will be formatted as a matrix of BLS scalar cells and then individual columns get erasure coded to ensure redundancy."#; let app_id_2_data = @@ -116,7 +116,7 @@ get erasure coded to ensure redundancy."#; let grid = EvaluationGrid::from_extrinsics(xts.clone(), 4, 32, 4, hash) .unwrap() - .extend_columns(unsafe { NonZeroU16::new_unchecked(2) }) + .extend_columns(unsafe { NonZeroU32::new_unchecked(2) }) .unwrap(); let bdims = grid.dims(); @@ -146,7 +146,7 @@ get erasure coded to ensure redundancy."#; #[test] fn test_extend_mock_data() { - let orig_data = br#"This is mocked test data. It will be formatted as a matrix of BLS scalar cells and then individual columns + let orig_data = br#"This is mocked test data. It will be formatted as a matrix of BLS scalar cells and then individual columns get erasure coded to ensure redundancy. Let's see how this gets encoded and then reconstructed by sampling only some data."#; let exts = vec![AppExtrinsic::from(orig_data.to_vec())]; @@ -155,7 +155,7 @@ Let's see how this gets encoded and then reconstructed by sampling only some dat let hash = Seed::default(); let grid = EvaluationGrid::from_extrinsics(exts.clone(), 4, 128, 2, hash) .unwrap() - .extend_columns(unsafe { NonZeroU16::new_unchecked(2) }) + .extend_columns(unsafe { NonZeroU32::new_unchecked(2) }) .unwrap(); let cols = sample_cells(&grid, None); diff --git a/kate/src/gridgen/tests/mod.rs b/kate/src/gridgen/tests/mod.rs index 30aa760..4c0558e 100644 --- a/kate/src/gridgen/tests/mod.rs +++ b/kate/src/gridgen/tests/mod.rs @@ -47,7 +47,7 @@ fn sample_unique(rng: &mut impl Rng, n_samples: usize, n: usize) -> Vec { fn sample_cells(grid: &EvaluationGrid, columns: Option>) -> Vec { let mut rng = ChaChaRng::from_seed([42u8; 32]); - let (g_rows, g_cols): (usize, usize) = grid.dims().into(); + let (g_rows, g_cols): (usize, usize) = grid.dims().as_usize(); let cols = columns.unwrap_or_else(|| (0..g_cols).collect()); cols.iter() diff --git a/kate/src/gridgen/tests/reconstruction.rs b/kate/src/gridgen/tests/reconstruction.rs index 1ac6f77..ce2edf3 100644 --- a/kate/src/gridgen/tests/reconstruction.rs +++ b/kate/src/gridgen/tests/reconstruction.rs @@ -5,7 +5,7 @@ use crate::{ Seed, }; use avail_core::{AppExtrinsic, AppId, BlockLengthColumns, BlockLengthRows}; -use core::num::NonZeroU16; +use core::num::NonZeroU32; use kate_recovery::{ com::{reconstruct_app_extrinsics, reconstruct_extrinsics}, data::SingleCell as DCell, @@ -28,11 +28,11 @@ fn test_multiple_extrinsics_for_same_app_id() { let hash = Seed::default(); let ev = EvaluationGrid::from_extrinsics(xts, 4, 128, 2, hash) .unwrap() - .extend_columns(unsafe { NonZeroU16::new_unchecked(2) }) + .extend_columns(unsafe { NonZeroU32::new_unchecked(2) }) .unwrap(); let cells = sample_cells(&ev, None); - let (rows, cols): (u16, u16) = ev.dims().into(); + let (rows, cols): (u32, u16) = ev.dims().into(); let bdims = Dimensions::new_from(rows, cols).unwrap(); let res = reconstruct_extrinsics(&ev.lookup, bdims, cells).unwrap(); @@ -44,8 +44,8 @@ proptest! { #![proptest_config(ProptestConfig::with_cases(5))] #[test] fn test_build_and_reconstruct(exts in super::app_extrinsics_strategy()) { - let grid = EvaluationGrid::from_extrinsics(exts.clone(), 4, 256, 256, Seed::default()).unwrap().extend_columns(unsafe { NonZeroU16::new_unchecked(2)}).unwrap(); - let (rows, cols) :(usize,usize)= grid.dims().into(); + let grid = EvaluationGrid::from_extrinsics(exts.clone(), 4, 256, 256, Seed::default()).unwrap().extend_columns(unsafe { NonZeroU32::new_unchecked(2)}).unwrap(); + let (rows, cols) :(usize,usize)= grid.dims().as_usize(); //let (layout, commitments, dims, matrix) = par_build_commitments( // BlockLengthRows(64), BlockLengthColumns(16), 32, xts, Seed::default()).unwrap(); const RNG_SEED: Seed = [42u8; 32]; @@ -100,7 +100,7 @@ get erasure coded to ensure redundancy."#; let grid = EvaluationGrid::from_extrinsics(xts, 4, 4, 32, Seed::default()) .unwrap() - .extend_columns(unsafe { NonZeroU16::new_unchecked(2) }) + .extend_columns(unsafe { NonZeroU32::new_unchecked(2) }) .unwrap(); let cols_1 = sample_cells(&grid, Some(vec![0, 1, 2, 3])); diff --git a/kate/src/lib.rs b/kate/src/lib.rs index 548533f..5b7f6b6 100644 --- a/kate/src/lib.rs +++ b/kate/src/lib.rs @@ -32,10 +32,10 @@ pub type M1NoPrecomp = pub type ArkScalar = Fr; pub mod config { use super::{BlockLengthColumns, BlockLengthRows}; - use core::num::NonZeroU16; + use core::num::{NonZeroU16, NonZeroU32}; pub const SCALAR_SIZE: usize = 32; - pub const ROW_EXTENSION: NonZeroU16 = unsafe { NonZeroU16::new_unchecked(2) }; + pub const ROW_EXTENSION: NonZeroU32 = unsafe { NonZeroU32::new_unchecked(2) }; pub const COL_EXTENSION: NonZeroU16 = NonZeroU16::MIN; pub const PROVER_KEY_SIZE: u32 = 48; pub const PROOF_SIZE: usize = 48;