Skip to content

Commit

Permalink
doc: Updated comments and documentation for melo-erasure-coding
Browse files Browse the repository at this point in the history
  • Loading branch information
@DarkingLee
DarkingLee committed Jul 27, 2023
1 parent 4bab713 commit ddea9bc
Show file tree
Hide file tree
Showing 4 changed files with 183 additions and 111 deletions.
9 changes: 5 additions & 4 deletions crates/melo-erasure-coding/src/extend_col.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ use crate::erasure_coding::{extend, extend_fs_g1};

/// Extends the segments in a column using FFT settings.
///
/// It extends the `segments` in the original column to twice their size, and also extends the `proof` in each `Segment`.
/// It extends the `segments` in the original column to twice their size, and also extends the `proof` in each
/// `Segment`.The homomorphic property of KZG commitments is used to extend the proof to the correct commitment of the
/// row where the data is located. This avoids the cost of recalculating commitments and proofs.
///
/// # Arguments
///
Expand All @@ -37,9 +39,8 @@ use crate::erasure_coding::{extend, extend_fs_g1};
///
/// # Notes
///
/// * The extended `Vec<Segment>` is not interleaved with parity data, and the `x` value of the `Position` in the original
/// data is not changed. This is to avoid confusion during the erasure coding process. Additionally, we do not make any
/// promises or validations in the column direction, so there is no need to do so.
/// * The extended `Vec<Segment>` is not interleaved with parity data, and the `y` value of the `Position` in the original
/// data is not changed. This is to avoid confusion during the erasure coding process.
pub fn extend_segments_col(
fs: &FsFFTSettings,
segments: &Vec<Segment>,
Expand Down
75 changes: 50 additions & 25 deletions crates/melo-erasure-coding/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,31 +27,56 @@ pub mod extend_col;
pub mod recovery;
pub mod segment;

/// Converts a vector of byte vectors `bytes_vec` into a vector of `Blob`s, where each `Blob` contains
/// `field_elements_per_blob` field elements.
///
/// # Arguments
///
/// * `bytes_vec` - A vector of byte vectors to convert.
/// * `field_elements_per_blob` - The number of field elements to include in each `Blob`.
///
/// # Returns
///
/// * `Result<Vec<Blob>, String>` - A vector of `Blob`s, or an error message if the conversion fails.
///
/// # Errors
///
/// Returns an error message if:
///
/// * `bytes_vec` contains an empty byte vector.
/// * `field_elements_per_blob` is not a power of two.
/// * `field_elements_per_blob` is zero.
///
/// # Notes
///
/// * If the input byte vectors do not contain enough bytes to create a `Blob`, the remaining bytes
/// are padded with zeroes. When using this function, the final recovered data should be determined
/// based on the length of the original data.
pub fn bytes_vec_to_blobs(
bytes_vec: &Vec<Vec<u8>>,
field_elements_per_blob: usize,
bytes_vec: &Vec<Vec<u8>>,
field_elements_per_blob: usize,
) -> Result<Vec<Blob>, String> {
if bytes_vec.iter().any(|bytes| bytes.is_empty()) {
return Err("bytes_vec should not contain empty bytes; qed".to_string());
}
if !field_elements_per_blob.is_power_of_two() {
return Err("field_elements_per_blob should be a power of 2; qed".to_string());
}
if field_elements_per_blob == 0 {
return Err("field_elements_per_blob should be greater than 0; qed".to_string());
}
let bytes_per_blob = SCALAR_SAFE_BYTES * field_elements_per_blob;
let blobs = bytes_vec
.iter()
.flat_map(|bytes| {
bytes
.chunks(bytes_per_blob)
.map(|chunk| {
Blob::try_from_bytes_pad(chunk, bytes_per_blob)
.expect("Failed to convert bytes to Blob; qed")
})
.collect_vec()
})
.collect_vec();
Ok(blobs)
if bytes_vec.iter().any(|bytes| bytes.is_empty()) {
return Err("bytes_vec should not contain empty bytes; qed".to_string());
}
if !field_elements_per_blob.is_power_of_two() {
return Err("field_elements_per_blob should be a power of 2; qed".to_string());
}
if field_elements_per_blob == 0 {
return Err("field_elements_per_blob should be greater than 0; qed".to_string());
}
let bytes_per_blob = SCALAR_SAFE_BYTES * field_elements_per_blob;
let blobs = bytes_vec
.iter()
.flat_map(|bytes| {
bytes
.chunks(bytes_per_blob)
.map(|chunk| {
Blob::try_from_bytes_pad(chunk, bytes_per_blob)
.expect("Failed to convert bytes to Blob; qed")
})
.collect_vec()
})
.collect_vec();
Ok(blobs)
}
75 changes: 41 additions & 34 deletions crates/melo-erasure-coding/src/recovery.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,49 +20,56 @@ use melo_core_primitives::kzg::{Position, KZG};
use melo_core_primitives::segment::{Segment, SegmentData};
use rust_kzg_blst::utils::reverse_bit_order;

/// Recover a row of segments from a vector of segments, using the provided KZG instance and chunk count.
///
/// # Arguments
///
/// * `segments` - A vector of `Segment`s to recover a row from.
/// * `kzg` - A `KZG` instance to use for recovery.
/// * `chunk_count` - The number of segments in the original data.
pub fn recovery_row_from_segments(
segments: &Vec<Segment>,
kzg: &KZG,
chunk_count: usize,
segments: &Vec<Segment>,
kzg: &KZG,
chunk_count: usize,
) -> Result<Vec<Segment>, String> {
let y = segments[0].position.y;
let segments_size = segments[0].size();
let y = segments[0].position.y;
let segments_size = segments[0].size();

if segments.iter().any(|s| s.position.y != y) {
return Err("segments are not from the same row".to_string());
}
if !segments_size.is_power_of_two() || !chunk_count.is_power_of_two() {
return Err("segment size and chunk_count must be a power of two".to_string());
}
if segments.iter().any(|s| s.position.y != y) {
return Err("segments are not from the same row".to_string());
}
if !segments_size.is_power_of_two() || !chunk_count.is_power_of_two() {
return Err("segment size and chunk_count must be a power of two".to_string());
}
if segments.iter().any(|s| s.size() != segments_size) {
return Err("segments are not of the same size".to_string());
}

let order_segments = order_segments_row(&segments, chunk_count)?;
let mut row = segment_datas_to_row(&order_segments, segments_size);
reverse_bit_order(&mut row);
let poly = recover_poly(kzg.get_fs(), &row)?;
let order_segments = order_segments_row(&segments, chunk_count)?;
let mut row = segment_datas_to_row(&order_segments, segments_size);
reverse_bit_order(&mut row);
let poly = recover_poly(kzg.get_fs(), &row)?;

let recovery_row = extend_poly(kzg.get_fs(), &poly)?;
let recovery_row = extend_poly(kzg.get_fs(), &poly)?;

order_segments
.iter()
.enumerate()
.map(|(i, segment_data)| {
let position = Position { x: i as u32, y };
match segment_data {
Some(segment_data) => Ok(Segment { position, content: segment_data.clone() }),
None => {
let index = i * segments_size;
let data = recovery_row[index..(i + 1) * segments_size].to_vec();
let segment_data =
SegmentData::from_data(&position, &data, kzg, &poly, segments.len())
.map_err(|e| e.to_string())?;
Ok(Segment { position, content: segment_data })
},
}
})
.collect()
order_segments
.iter()
.enumerate()
.map(|(i, segment_data)| {
let position = Position { x: i as u32, y };
match segment_data {
Some(segment_data) => Ok(Segment { position, content: segment_data.clone() }),
None => {
let index = i * segments_size;
let data = recovery_row[index..(i + 1) * segments_size].to_vec();
let segment_data =
SegmentData::from_data(&position, &data, kzg, &poly, segments.len())
.map_err(|e| e.to_string())?;
Ok(Segment { position, content: segment_data })
},
}
})
.collect()
}

// TODO
Expand Down
135 changes: 87 additions & 48 deletions crates/melo-erasure-coding/src/segment.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,37 +20,70 @@ use rust_kzg_blst::types::fk20_multi_settings::FsFK20MultiSettings;

use crate::erasure_coding::extend_poly;

/// Orders a vector of `Segment`s into a row of `SegmentData` using the provided chunk count.
///
/// The returned vector is of type `Vec<Option<SegmentData>>`, where `Option<SegmentData>` is an `Option` type.
/// If there is data at a position, it is `Some`, otherwise it is `None`.The final length of the returned vector
/// is `chunk_count * 2`, where `chunk_count` is the length of the original data.
///
/// # Arguments
///
/// * `segments` - A vector of `Segment`s to order into a row.
/// * `chunk_count` - The number of chunks to use for ordering.
///
/// # Returns
///
/// A `Result` containing a vector of `Option<SegmentData>` representing the ordered row, or an error message
/// if ordering fails.
pub fn order_segments_row(segments: &Vec<Segment>, chunk_count: usize) -> Result<Vec<Option<SegmentData>>, String> {
if segments.len() > chunk_count * 2 || segments.len() == 0 {
return Err("segments x not equal".to_string());
}
let y = segments[0].position.y;
let mut ordered_segments = vec![None; chunk_count * 2];
for segment in segments.iter() {
if segment.position.y != y {
return Err("segments y not equal".to_string());
}
ordered_segments[segment.position.x as usize] = Some(segment.content.clone());
}
Ok(ordered_segments)
if segments.len() > chunk_count * 2 || segments.len() == 0 {
return Err("segments x not equal".to_string());
}
let y = segments[0].position.y;
let mut ordered_segments = vec![None; chunk_count * 2];
for segment in segments.iter() {
if segment.position.y != y {
return Err("segments y not equal".to_string());
}
ordered_segments[segment.position.x as usize] = Some(segment.content.clone());
}
Ok(ordered_segments)
}

/// Orders segments by column.
///
/// The returned vector is of type `Vec<Option<SegmentData>>`, where `Option<SegmentData>` is an `Option` type.
/// If there is data at a position, it is `Some`, otherwise it is `None`. The final length of the returned vector
/// is `k * 2`, where `k` is the number of segments in the original data.
///
/// # Arguments
///
/// * `segments` - A vector of `Segment` structs to be ordered.
/// * `k` - The number of segments to be ordered.
///
/// # Returns
///
/// A `Result` containing a vector of `Option<SegmentData>` structs if successful, or a `String` error message if unsuccessful.
///
/// # Errors
///
/// Returns an error message if the length of `segments` is greater than `k * 2` or if the length of `segments` is 0.
pub fn order_segments_col(
segments: &Vec<Segment>,
k: usize,
segments: &Vec<Segment>,
k: usize,
) -> Result<Vec<Option<SegmentData>>, String> {
if segments.len() > k * 2 || segments.len() == 0 {
return Err("segments x not equal".to_string());
}
let x = segments[0].position.x;
let mut ordered_segments = vec![None; k * 2];
for segment in segments.iter() {
if segment.position.x != x {
return Err("segments x not equal".to_string());
}
ordered_segments[segment.position.y as usize] = Some(segment.content.clone());
}
Ok(ordered_segments)
if segments.len() > k * 2 || segments.len() == 0 {
return Err("segments x not equal".to_string());
}
let x = segments[0].position.x;
let mut ordered_segments = vec![None; k * 2];
for segment in segments.iter() {
if segment.position.x != x {
return Err("segments x not equal".to_string());
}
ordered_segments[segment.position.y as usize] = Some(segment.content.clone());
}
Ok(ordered_segments)
}

pub fn segment_datas_to_row(segments: &Vec<Option<SegmentData>>, chunk_size: usize) -> Vec<Option<BlsScalar>> {
Expand All @@ -71,32 +104,38 @@ pub fn segment_datas_to_row(segments: &Vec<Option<SegmentData>>, chunk_size: usi
///
/// # Arguments
///
/// * `poly` - A reference to a `Polynomial` struct.
/// * `fk` - A reference to a `FsFK20MultiSettings` struct.
/// * `poly` - A reference to a `Polynomial` struct to be converted. The length of the polynomial must be a power of two.
/// * `kzg` - A reference to a `KZG` struct.
/// * `y` - The y-coordinate of the position of the segment.
/// * `chunk_size` - The size of each chunk. Must be a power of two.
///
/// # Returns
///
/// A `Result` containing a vector of `Segment` structs or an error message.
/// A `Result` containing a vector of `Segment` structs if successful, or a `String` error message if unsuccessful.
///
/// # Errors
///
/// Returns an error message if `chunk_size` is not a power of two.
pub fn poly_to_segment_vec(poly: &Polynomial, kzg: &KZG, y: usize, chunk_size: usize) -> Result<Vec<Segment>, String> {
let poly_len = poly.checked()?.0.coeffs.len();
let poly_len = poly.checked()?.0.coeffs.len();

// chunk_size must be a power of two
if !chunk_size.is_power_of_two() {
return Err("chunk_size must be a power of two".to_string());
}
let fk = FsFK20MultiSettings::new(&kzg.ks, 2 * poly_len, chunk_size).unwrap();
let all_proofs = fk.data_availability(&poly.0).unwrap();
let extended_poly = extend_poly(&fk.kzg_settings.fs, &poly)?;
let segments = extended_poly
.chunks(chunk_size)
.enumerate()
.map(|(i, chunk)| {
let position = Position { y: y as u32, x: i as u32 };
let proof = all_proofs[i];
Segment::new(position, chunk, KZGProof(proof))
})
.collect::<Vec<_>>();
// chunk_size must be a power of two
if !chunk_size.is_power_of_two() {
return Err("chunk_size must be a power of two".to_string());
}

let fk = FsFK20MultiSettings::new(&kzg.ks, 2 * poly_len, chunk_size).unwrap();
let all_proofs = fk.data_availability(&poly.0).unwrap();
let extended_poly = extend_poly(&fk.kzg_settings.fs, &poly)?;
let segments = extended_poly
.chunks(chunk_size)
.enumerate()
.map(|(i, chunk)| {
let position = Position { y: y as u32, x: i as u32 };
let proof = all_proofs[i];
Segment::new(position, chunk, KZGProof(proof))
})
.collect::<Vec<_>>();

Ok(segments)
Ok(segments)
}

0 comments on commit ddea9bc

Please sign in to comment.