Scrubber review

.github/workflows/rust.yml

@@ -0,0 +1,24 @@
+name: Build and test
+
+on: [ push ]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Formatting
+        run: cargo fmt -- --check
+      - name: Clippy
+        run: cargo clippy --all-targets --tests -- -D warnings
+      - name: Build
+        run: cargo build --all-features --verbose
+      - name: Run tests
+        run: cargo test --verbose
+      - name: Run binary
+        run: cargo run -p runner --verbose --release

Cargo.toml

@@ -0,0 +1,21 @@
+[workspace]
+members = ["runner"]
+
+[package]
+name = "sbc_algorithm"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+chunkfs = { git = "https://github.com/Piletskii-Oleg/chunkfs.git" }
+rayon = "1.10"
+zstd = "0.13"
+thiserror = "2.0.12"
+huffman-compress = "0.6.1"
+bit-vec = "0.6.3"
+log = "0.4.27"
+fasthash = "0.4.0"
+
+[dev-dependencies]
+chunkfs = { git = "https://github.com/Piletskii-Oleg/chunkfs.git", features = ["chunkers", "hashers"]}
+rand = "0.8.5"

README.md

@@ -0,0 +1,59 @@
+## Similarity Based Chunking Scrubber
+SBC Scrubber is a scrubber that can be used to implement different SBC algorithms with ChunkFS
+
+SBC Scrubber is currently under active development, breaking changes can always happen.
+
+## Usage
+
+Add the following dependency to your `Cargo.toml`:
+
+```toml
+[dependencies]
+chunkfs = { version = "0.1", features = ["chunkers", "hashers"] }
+sbc_algorithm = { git = "https://github.com/maxscherbakov/sbc_algorithm.git" }
+```
+
+## Example
+
+```rust
+extern crate chunkfs;
+extern crate sbc_algorithm;
+
+use chunkfs::chunkers::{SizeParams, SuperChunker};
+use chunkfs::hashers::Sha256Hasher;
+use chunkfs::FileSystem;
+use sbc_algorithm::{clusterer, decoder, encoder, hasher};
+use sbc_algorithm::{SBCMap, SBCScrubber};
+use std::collections::HashMap;
+use std::{fs, io};
+
+fn main() -> io::Result<()> {
+    let data = vec![10; 1024 * 1024];
+    let chunk_size = SizeParams::new(2 * 1024, 8 * 1024, 16 * 1024);
+    let mut fs = FileSystem::new_with_scrubber(
+        HashMap::default(),
+        SBCMap::new(decoder::GdeltaDecoder::new(false)),
+        Box::new(SBCScrubber::new(
+            hasher::AronovichHasher,
+            clusterer::GraphClusterer::default(),
+            encoder::GdeltaEncoder::new(false),
+        )),
+        Sha256Hasher::default(),
+    );
+    let mut handle = fs.create_file("file".to_string(), SuperChunker::new(chunk_size))?;
+    fs.write_to_file(&mut handle, &data)?;
+    fs.close_file(handle)?;
+
+    let read_handle = fs.open_file_readonly("file")?;
+    let read = fs.read_file_complete(&read_handle)?;
+
+    let cdc_dedup_ratio = fs.cdc_dedup_ratio();
+    let res = fs.scrub().unwrap();
+    let sbc_dedup_ratio = fs.total_dedup_ratio();
+    println!("CDC dedup ratio: {}", cdc_dedup_ratio);
+    println!("SBC dedup ratio: {}", sbc_dedup_ratio);
+    println!("ScrubMeasure: {:?}", res);
+    assert_eq!(read.len(), data.len());
+    Ok(())
+}
+```

runner/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "runner"
+version = "0.1.0"
+edition = "2021"
+
+
+[dependencies]
+sbc_algorithm = { path = ".." }
+chunkfs = { git = "https://github.com/Piletskii-Oleg/chunkfs.git", features = ["chunkers", "hashers"]}

runner/src/main.rs

@@ -0,0 +1,40 @@
+extern crate chunkfs;
+extern crate sbc_algorithm;
+
+use chunkfs::chunkers::{SizeParams, SuperChunker};
+use chunkfs::hashers::Sha256Hasher;
+use chunkfs::FileSystem;
+use sbc_algorithm::{clusterer, decoder, encoder, hasher};
+use sbc_algorithm::{SBCMap, SBCScrubber};
+use std::collections::HashMap;
+use std::{fs, io};
+
+fn main() -> io::Result<()> {
+    let data = fs::read("runner/files/my_data")?;
+    let chunk_size = SizeParams::new(2 * 1024, 8 * 1024, 16 * 1024);
+    let mut fs = FileSystem::new_with_scrubber(
+        HashMap::default(),
+        SBCMap::new(decoder::GdeltaDecoder::new(false)),
+        Box::new(SBCScrubber::new(
+            hasher::AronovichHasher,
+            clusterer::GraphClusterer::default(),
+            encoder::GdeltaEncoder::new(false),
+        )),
+        Sha256Hasher::default(),
+    );
+    let mut handle = fs.create_file("file".to_string(), SuperChunker::new(chunk_size))?;
+    fs.write_to_file(&mut handle, &data)?;
+    fs.close_file(handle)?;
+
+    let read_handle = fs.open_file_readonly("file")?;
+    let read = fs.read_file_complete(&read_handle)?;
+
+    let cdc_dedup_ratio = fs.cdc_dedup_ratio();
+    let res = fs.scrub().unwrap();
+    let sbc_dedup_ratio = fs.total_dedup_ratio();
+    println!("CDC dedup ratio: {}", cdc_dedup_ratio);
+    println!("SBC dedup ratio: {}", sbc_dedup_ratio);
+    println!("ScrubMeasure: {:?}", res);
+    assert_eq!(read.len(), data.len());
+    Ok(())
+}