app: add --verbose and re-use chunker

seedvault-app · Jul 11, 2024 · dfa6241 · dfa6241
1 parent 9789d29
commit dfa6241
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 16 deletions.
diff --git a/app/src/main/java/org/calyxos/seedvault/chunker/Main.kt b/app/src/main/java/org/calyxos/seedvault/chunker/Main.kt
@@ -19,29 +19,34 @@ class Cli : CliktCommand() {
     private val files by argument().multiple(required = true)
 
     private val checkDedupRatio: Boolean by option().flag(default = false)
+    private val verbose: Boolean by option("-v", "--verbose").flag(default = false)
 
     private val size: Int by option()
         .int()
         .restrictTo(AVERAGE_MIN..AVERAGE_MAX)
-        .default(16384)
+        .default(16 * 1024)
 
     private val normalization: Int by option()
         .int()
         .restrictTo(0..3)
         .default(1)
 
     override fun run() {
+        val digest = MessageDigest.getInstance("SHA-256")
+        val chunker = Chunker(size, normalization) { bytes ->
+            digest.digest(bytes).fold("") { str, it -> str + "%02x".format(it) }
+        }
         val duration = measureTime {
             files.forEach { file ->
-                onEachFile(File(file))
+                onEachFile(chunker, File(file))
             }
         }
         println("\nTook: $duration")
         if (checkDedupRatio) {
             println()
             val totalSize = files.sumOf { File(it).length() }
-            val sizePerTime = totalSize / duration.inWholeSeconds
-            println("Files: ${files.size} with a total of $totalSize bytes ($sizePerTime bytes/s)")
+            val sizePerTime = totalSize / duration.inWholeSeconds / 1024 / 1024
+            println("Files: ${files.size} with a total of $totalSize bytes ($sizePerTime MiB/s)")
             println("Unique chunks: ${chunks.size}")
             println("Dupe chunks: $reusedChunks")
             println("Dupe data: $sizeDupe")
@@ -53,22 +58,20 @@ class Cli : CliktCommand() {
     private var reusedChunks: Int = 0
     private var sizeDupe: Long = 0L
 
-    private fun onEachFile(file: File) {
-        println()
-        println(file.absolutePath)
+    private fun onEachFile(chunker: Chunker, file: File) {
+        if (verbose) {
+            println()
+            println(file.absolutePath)
+        }
         if (!file.isFile) {
-            println("  not a file, ignoring...")
+            if (verbose) println("  not a file, ignoring...")
             return
         }
-        val digest = MessageDigest.getInstance("SHA-256")
-        val chunker = Chunker(size, normalization) { bytes ->
-            digest.digest(bytes).fold("") { str, it -> str + "%02x".format(it) }
-        }
         chunker.chunk(file) { chunk -> onNewChunk(chunk) }
     }
 
     private fun onNewChunk(chunk: Chunk) {
-        println("hash=${chunk.hash} offset=${chunk.offset} size=${chunk.length}")
+        if (verbose) println("hash=${chunk.hash} offset=${chunk.offset} size=${chunk.length}")
         if (checkDedupRatio) {
             if (chunk.hash in chunks) {
                 sizeDupe += chunk.length

diff --git a/lib/src/main/java/org/calyxos/seedvault/chunker/Chunker.kt b/lib/src/main/java/org/calyxos/seedvault/chunker/Chunker.kt
@@ -10,6 +10,7 @@ import org.calyxos.seedvault.chunker.GearTableCreator.GEAR_SIZE
 import java.io.File
 import java.lang.Byte.toUnsignedInt
 import kotlin.math.min
+import kotlin.math.roundToInt
 
 class Chunk(val offset: Long, val length: Int, val data: ByteArray, val hash: String)
 
@@ -34,10 +35,10 @@ class Chunker(
     private val blob = RingByteArray(maxSize * 2)
     private var offset: Long = 0
 
-    constructor(avgSize: Int, normalization: Int = 2, hashFunction: (ByteArray) -> String) : this(
-        minSize = avgSize.floorDiv(4),
+    constructor(avgSize: Int, normalization: Int = 1, hashFunction: (ByteArray) -> String) : this(
+        minSize = avgSize.floorDiv(2),
         avgSize = avgSize,
-        maxSize = avgSize * 8,
+        maxSize = (avgSize * 2.5).roundToInt(),
         normalization = normalization,
         gearTable = Const.GEAR,
         hashFunction = hashFunction,