Wp/ updated delete file to use pk as filename

src/backend.rs

@@ -1,2 +1 @@
 pub mod fs;
-// pub mod task;

src/backend/fs.rs

@@ -134,6 +134,7 @@ pub fn write_segment(sk: &[u8], pk: &[u8], encoded: &Encoded) -> Result<BaoHash>
 
 pub fn write_catalog(file_hash: &Blake3Hash, segment_hashes: &[BaoHash]) -> Result<()> {
     debug!("Write catalog");
+
     let contents: Vec<u8> = segment_hashes
         .iter()
         .flat_map(|bao_hash| bao_hash.to_bytes())
@@ -255,7 +256,56 @@ pub fn read_catalog(file_hash: &Blake3Hash) -> Result<Vec<BaoHash>> {
     Ok(bao_hashes)
 }
 
-#[allow(unused_variables)]
+pub async fn read_slices(
+    hash: bao::Hash, // from Blake3
+    file_bytes: &[u8],
+    slice_start: u64, //let slice_start = 65536;
+    slice_len: u16,   // let slice_len = 8192;
+) -> Result<Vec<u8>> {
+    use std::io::prelude::*;
+
+    // Start by encoding some input.
+    let (encoded_input, hash) = bao::encode::encode(file_bytes);
+    debug!(">>> encoded INPUT LENGTH:: {:?}", &encoded_input.len());
+    debug!(">>> fs hash hash:: {:?}", &hash);
+
+    // Slice the encoding. These parameters are multiples of the chunk size, which avoids
+    // unnecessary overhead.
+    let slice_start = 65536;
+    let slice_len = 8192;
+    let encoded_cursor = std::io::Cursor::new(&encoded_input);
+    let mut extractor = bao::encode::SliceExtractor::new(encoded_cursor, slice_start, slice_len);
+    let mut slice = Vec::new();
+    extractor.read_to_end(&mut slice)?;
+
+    // Decode the slice. The result should be the same as the part of the input that the slice
+    // represents. Note that we're using the same hash that encoding produced, which is
+    // independent of the slice parameters. That's the whole point; if we just wanted to re-encode
+    // a portion of the input and wind up with a different hash, we wouldn't need slicing.
+    let mut decoded = Vec::new();
+    let mut decoder = bao::decode::SliceDecoder::new(&*slice, &hash, slice_start, slice_len);
+    decoder.read_to_end(&mut decoded)?;
+    // assert_eq!(
+    //     &encoded_input[slice_start as usize..][..slice_len as usize],
+    //     &*decoded
+    // );
+    debug!(
+        "usize vs length: {:?}",
+        &encoded_input[slice_start as usize..][..slice_len as usize].len()
+    );
+
+    // Like regular decoding, slice decoding will fail if the hash doesn't match.
+    let mut bad_slice = slice.clone();
+    debug!("BAD SLICE");
+    let last_index = bad_slice.len() - 1;
+    bad_slice[last_index] ^= 1;
+    let mut decoder = bao::decode::SliceDecoder::new(&*bad_slice, &hash, slice_start, slice_len);
+    let err = decoder.read_to_end(&mut Vec::new()).unwrap_err();
+    assert_eq!(std::io::ErrorKind::InvalidData, err.kind());
+
+    Ok(decoded)
+}
+
 pub fn delete_file(pk: Secp256k1PubKey, file_bytes: &[u8]) -> Result<()> {
     let pk_bytes = pk.to_bytes();
     let (x_only_pk, _) = pk.into_inner().x_only_public_key();
@@ -277,9 +327,8 @@ pub fn delete_file(pk: Secp256k1PubKey, file_bytes: &[u8]) -> Result<()> {
     Ok(())
 }
 
-#[allow(unused_variables)]
+// TODO: These aren't worth breaking out into their own functions
 fn remove_dir_contents<P: AsRef<Path>>(path: P, seg_file: PathBuf) -> io::Result<()> {
-    trace!(">>> remove_Segment_contents");
     for entry in fs::read_dir(path)? {
         trace!("Delete Segment File at {:?}", entry);
         fs::remove_file(entry?.path())?;
@@ -295,33 +344,3 @@ fn remove_dir_catalogs(path: PathBuf, file: PathBuf) -> io::Result<()> {
     }
     Ok(())
 }
-
-// fn remove_dir_segements<P: AsRef<Path>>(path: P, seg_file: PathBuf) -> io::Result<()> {
-//     trace!(">>> remove_Segment_contents");
-//     for entry in fs::read_dir(path)? {
-//         let entry = entry?;
-//         trace!("ENTRY Delete SEGMENT File at {:?}", entry);
-
-//         match &entry {
-//             seg_file => {
-//                 fs::remove_file(seg_file.path())?;
-//                 trace!("Delete Segment File at {:?}", seg_file);
-//             }
-//         }
-//     }
-//     Ok(())
-// }
-
-// fn remove_dir_catalogs(path: PathBuf, file: PathBuf) -> io::Result<()> {
-//     for entry in fs::read_dir(path)? {
-//         let entry = entry?;
-//         trace!("ENTRY Delete CATALOG File at {:?}", entry);
-//         match &entry {
-//             file => {
-//                 fs::remove_file(file.path())?;
-//                 trace!("FILE MATCH Delete CATALOG File at {:?}", file);
-//             }
-//         }
-//     }
-//     Ok(())
-// }

src/frontend/http.rs

@@ -1,9 +1,9 @@
-use std::{net::SocketAddr, str::FromStr};
+use std::{net::SocketAddr, str::FromStr, sync::RwLock};
 
 use anyhow::Result;
 use axum::{
     body::StreamBody,
-    extract::{BodyStream, Path},
+    extract::{BodyStream, Path, Query},
     http::StatusCode,
     response::{IntoResponse, Response},
     routing::{delete, get, post},
@@ -13,10 +13,11 @@ use bytes::BytesMut;
 use futures_util::{stream, StreamExt};
 use log::{debug, info, trace};
 use secp256k1::PublicKey;
+use serde::Deserialize;
 use tower_http::cors::CorsLayer;
 
 use crate::{
-    backend::fs::{delete_file, read_file, write_file, FileStream},
+    backend::fs::{delete_file, read_file, read_slices, write_file, FileStream},
     config::{node_shared_secret, SYS_CFG},
     prelude::*,
 };
@@ -58,6 +59,15 @@ async fn post_file(
     Ok((StatusCode::OK, hash.to_hex().to_string()))
 }
 
+/* TODO: This seems unnecessary, and could introduce concurrency and security issues
+#[axum_macros::debug_handler]
+async fn get_last_hash(Path(_pk): Path<String>) -> Result<impl IntoResponse, AppError> {
+    let hash1 = LOCK.read().unwrap();
+    let string_hash1 = hash1.clone();
+
+    Ok((StatusCode::OK, string_hash1))
+} */
+
 #[axum_macros::debug_handler]
 async fn get_file(
     Path((pk, blake3_hash)): Path<(String, String)>,
@@ -91,7 +101,9 @@ pub async fn start() -> Result<()> {
     let app = Router::new()
         .route("/remove/:pk/:blake3_hash", delete(remove_file))
         .route("/store/:pk", post(post_file))
+        // .route("/last_hash/:pk", get(get_last_hash))
         .route("/retrieve/:pk/:blake3_hash", get(get_file))
+        // .route("/slice/", get(get_slice))
         .route("/key/:pk", get(key))
         // .route("/catalog/:blake3_hash", get(get_catalog))
         // .route("/raw/:bao_hash", get(get_raw))

src/structs.rs

@@ -12,6 +12,16 @@ impl fmt::Display for Blake3Hash {
     }
 }
 
+impl TryFrom<&[u8]> for Blake3Hash {
+    type Error = Error;
+
+    fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
+        let mut hash = [0_u8; 32];
+        hash.copy_from_slice(&value[0..32]);
+        Ok(Self(blake3::Hash::try_from(hash)?))
+    }
+}
+
 pub struct BaoHash(pub bao::Hash);
 
 impl BaoHash {

tests/file.rs

@@ -72,40 +72,6 @@ async fn write_read() -> Result<()> {
     Ok(())
 }
 
-#[tokio::test]
-async fn check_catalog_exists() -> Result<()> {
-    carbonado::utils::init_logging(RUST_LOG);
-
-    let (_sk, pk) = generate_keypair(&mut thread_rng());
-
-    info!("Reading file bytes");
-    let file_bytes = fs::read("tests/samples/cat.gif")?;
-    debug!("{} bytes read", file_bytes.len());
-
-    let file_stream = stream::iter(file_bytes)
-        .chunks(1024 * 1024)
-        .map(|chunk| Ok(Bytes::from(chunk)))
-        .boxed();
-
-    info!("Writing file if not exists");
-    let is_ok = write_file(&Secp256k1PubKey(pk), file_stream).await.is_ok();
-    debug!("Skip writing file as File hash exists: {is_ok}");
-    assert!(is_ok);
-
-    let file_bytes = fs::read("tests/samples/cat.gif")?;
-    let file_stream = stream::iter(file_bytes)
-        .chunks(1024 * 1024)
-        .map(|chunk| Ok(Bytes::from(chunk)))
-        .boxed();
-
-    info!("Writing file if not exists");
-    let is_err = write_file(&Secp256k1PubKey(pk), file_stream).await.is_err();
-    debug!("Skip writing file as File hash exists: {is_err}");
-    assert!(is_err);
-
-    Ok(())
-}
-
 #[tokio::test]
 async fn read_write_delete_file() -> Result<()> {
     carbonado::utils::init_logging(RUST_LOG);
@@ -114,8 +80,8 @@ async fn read_write_delete_file() -> Result<()> {
     let file_bytes = fs::read("tests/samples/cat.gif")?;
     debug!("{} Write Delete:: bytes read", file_bytes.len());
 
-    let file_stream = stream::iter(file_bytes.clone())
-        .chunks(1024 * 1024)
+    let file_stream: FileStream = stream::iter(file_bytes.clone())
+        .chunks(SEGMENT_SIZE)
         .map(|chunk| Ok(Bytes::from(chunk)))
         .boxed();
 
@@ -135,8 +101,8 @@ async fn read_write_delete_file() -> Result<()> {
     // second file to write
     let (_sk, pk) = generate_keypair(&mut thread_rng());
 
-    let file_stream = stream::iter(file_bytes)
-        .chunks(1024 * 1024)
+    let file_stream: FileStream = stream::iter(file_bytes)
+        .chunks(SEGMENT_SIZE)
         .map(|chunk| Ok(Bytes::from(chunk)))
         .boxed();