Skip to content

Commit

Permalink
Fully implement SnapshotManager
Browse files Browse the repository at this point in the history 
which manages interactions with snapshots, such as loading, saving and removing them.
It also keeps a reference to the latestSnapshot that holds important re-usable data.
  • Loading branch information
@grote
grote committed Sep 13, 2024
1 parent fa1579e commit 2802ca6
Show file tree
Hide file tree
Showing 2 changed files with 124 additions and 21 deletions.
16 changes: 12 additions & 4 deletions app/src/main/java/com/stevesoltys/seedvault/transport/SnapshotManager.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,16 @@ internal class SnapshotManager(
* currently available on the backend.
*/
suspend fun onSnapshotsLoaded(handles: List<AppBackupFileType.Snapshot>): List<Snapshot> {
return handles.map { snapshotHandle ->
// TODO set up local snapshot cache, so we don't need to download those all the time
// TODO is it a fatal error when one snapshot is corrupted or couldn't get loaded?
val snapshot = loadSnapshot(snapshotHandle)
return handles.mapNotNull { snapshotHandle ->
val snapshot = try {
loadSnapshot(snapshotHandle)
} catch (e: Exception) {
// This isn't ideal, but the show must go on and we take the snapshots we can get.
// After the first load, a snapshot will get cached, so we are not hitting backend.
// TODO use a re-trying backend for snapshot loading
log.error(e) { "Error loading snapshot: $snapshotHandle" }
return@mapNotNull null
}
// update latest snapshot if this one is more recent
if (snapshot.token > (latestSnapshot?.token ?: 0)) latestSnapshot = snapshot
snapshot
Expand Down Expand Up @@ -87,6 +93,7 @@ internal class SnapshotManager(
* Removes the snapshot referenced by the given [snapshotHandle] from the backend
* and local cache.
*/
@Throws(IOException::class)
suspend fun removeSnapshot(snapshotHandle: AppBackupFileType.Snapshot) {
backendManager.backend.remove(snapshotHandle)
// remove from cache as well
Expand All @@ -97,6 +104,7 @@ internal class SnapshotManager(
* Loads and parses the snapshot referenced by the given [snapshotHandle].
* If a locally cached version exists, the backend will not be hit.
*/
@Throws(IOException::class)
private suspend fun loadSnapshot(snapshotHandle: AppBackupFileType.Snapshot): Snapshot {
val file = File(snapshotFolder, snapshotHandle.name)
val inputStream = if (file.isFile) {
Expand Down
129 changes: 112 additions & 17 deletions app/src/test/java/com/stevesoltys/seedvault/transport/SnapshotManagerTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,30 @@
package com.stevesoltys.seedvault.transport

import com.stevesoltys.seedvault.backend.BackendManager
import com.stevesoltys.seedvault.proto.snapshot
import com.stevesoltys.seedvault.transport.restore.Loader
import io.mockk.Runs
import io.mockk.coEvery
import io.mockk.coVerify
import io.mockk.every
import io.mockk.just
import io.mockk.mockk
import io.mockk.slot
import kotlinx.coroutines.runBlocking
import org.calyxos.seedvault.core.backends.AppBackupFileType
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.toByteArrayFromHex
import org.calyxos.seedvault.core.toHexString
import org.junit.jupiter.api.Assertions.assertArrayEquals
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.io.TempDir
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.io.File
import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
import java.nio.file.Path
Expand All @@ -33,29 +40,103 @@ internal class SnapshotManagerTest : TransportTest() {

private val backendManager: BackendManager = mockk()
private val backend: Backend = mockk()
private val loader: Loader = mockk()

private val loader = Loader(crypto, backendManager) // need a real loader

private val messageDigest = MessageDigest.getInstance("SHA-256")
private val ad = Random.nextBytes(1)
private val passThroughOutputStream = slot<OutputStream>()
private val passThroughInputStream = slot<InputStream>()
private val snapshotHandle = slot<AppBackupFileType.Snapshot>()

// @Test
// fun `test onSnapshotsLoaded sets latestSnapshot`(@TempDir tmpDir: Path) = runBlocking {
// val snapshotManager = getSnapshotManager(File(tmpDir.toString()))
//
// val snapshotHandle1 = AppBackupFileType.Snapshot(repoId, chunkId1)
// val snapshotHandle2 = AppBackupFileType.Snapshot(repoId, chunkId2)
// snapshotManager.onSnapshotsLoaded(listOf(snapshotHandle1, snapshotHandle2))
// Unit
// }
init {
every { backendManager.backend } returns backend
}

@Test
fun `test saving and loading`(@TempDir tmpDir: Path) = runBlocking {
fun `test onSnapshotsLoaded sets latestSnapshot`(@TempDir tmpDir: Path) = runBlocking {
val snapshotManager = getSnapshotManager(File(tmpDir.toString()))
val snapshotData1 = snapshot { token = 20 }.toByteArray()
val snapshotData2 = snapshot { token = 10 }.toByteArray()
val inputStream1 = ByteArrayInputStream(snapshotData1)
val inputStream2 = ByteArrayInputStream(snapshotData2)

val snapshotHandle1 = AppBackupFileType.Snapshot(repoId, chunkId1)
val snapshotHandle2 = AppBackupFileType.Snapshot(repoId, chunkId2)

coEvery { loader.loadFile(snapshotHandle1, any()) } returns inputStream1
coEvery { loader.loadFile(snapshotHandle2, any()) } returns inputStream2
snapshotManager.onSnapshotsLoaded(listOf(snapshotHandle1, snapshotHandle2))

// snapshot with largest token is latest
assertEquals(20, snapshotManager.latestSnapshot?.token)
}

@Test
fun `saveSnapshot saves to local cache`(@TempDir tmpDir: Path) = runBlocking {
val snapshotManager = getSnapshotManager(File(tmpDir.toString()))
val snapshotHandle = AppBackupFileType.Snapshot(repoId, chunkId1)
val outputStream = ByteArrayOutputStream()

every { crypto.getAdForVersion() } returns ad
every { crypto.newEncryptingStream(capture(passThroughOutputStream), ad) } answers {
passThroughOutputStream.captured // not really encrypting here
}
every { crypto.sha256(any()) } returns chunkId1.toByteArrayFromHex()
every { crypto.repoId } returns repoId
coEvery { backend.save(snapshotHandle) } returns outputStream

snapshotManager.saveSnapshot(snapshot)

val snapshotFile = File(tmpDir.toString(), snapshotHandle.name)
assertTrue(snapshotFile.isFile)
assertTrue(outputStream.size() > 0)
val cachedBytes = snapshotFile.inputStream().use { it.readAllBytes() }
assertArrayEquals(outputStream.toByteArray(), cachedBytes)
}

@Test
fun `snapshot loads from cache without backend`(@TempDir tmpDir: Path) = runBlocking {
val snapshotManager = getSnapshotManager(File(tmpDir.toString()))
val snapshotData = snapshot { token = 1337 }.toByteArray()
val inputStream = ByteArrayInputStream(snapshotData)
val snapshotHandle = AppBackupFileType.Snapshot(repoId, chunkId1)

// create cached file
val file = File(tmpDir.toString(), snapshotHandle.name)
file.outputStream().use { it.write(snapshotData) }

coEvery { loader.loadFile(file, snapshotHandle.hash) } returns inputStream

snapshotManager.onSnapshotsLoaded(listOf(snapshotHandle))

coVerify(exactly = 0) { // did not load from backend
loader.loadFile(snapshotHandle, any())
}
}

@Test
fun `failing to load a snapshot isn't fatal`(@TempDir tmpDir: Path) = runBlocking {
val snapshotManager = getSnapshotManager(File(tmpDir.toString()))

val snapshotData = snapshot { token = 42 }.toByteArray()
val inputStream = ByteArrayInputStream(snapshotData)

val snapshotHandle1 = AppBackupFileType.Snapshot(repoId, chunkId1)
val snapshotHandle2 = AppBackupFileType.Snapshot(repoId, chunkId2)

coEvery { loader.loadFile(snapshotHandle1, any()) } returns inputStream
coEvery { loader.loadFile(snapshotHandle2, any()) } throws IOException()
snapshotManager.onSnapshotsLoaded(listOf(snapshotHandle1, snapshotHandle2))

// still one snapshot survived and we didn't crash
assertEquals(42, snapshotManager.latestSnapshot?.token)
}

@Test
fun `test saving and loading`(@TempDir tmpDir: Path) = runBlocking {
val loader = Loader(crypto, backendManager) // need a real loader
val snapshotManager = getSnapshotManager(File(tmpDir.toString()), loader)

val messageDigest = MessageDigest.getInstance("SHA-256")
val bytes = slot<ByteArray>()
val outputStream = ByteArrayOutputStream()

Expand All @@ -64,16 +145,13 @@ internal class SnapshotManagerTest : TransportTest() {
passThroughOutputStream.captured // not really encrypting here
}
every { crypto.repoId } returns repoId
every { backendManager.backend } returns backend
every { crypto.sha256(capture(bytes)) } answers {
messageDigest.digest(bytes.captured)
}
coEvery { backend.save(capture(snapshotHandle)) } returns outputStream

snapshotManager.saveSnapshot(snapshot)

println(snapshotHandle.captured)

// check that file content hash matches snapshot hash
assertEquals(
messageDigest.digest(outputStream.toByteArray()).toHexString(),
Expand All @@ -96,7 +174,24 @@ internal class SnapshotManagerTest : TransportTest() {
}
}

private fun getSnapshotManager(tmpFolder: File): SnapshotManager {
@Test
fun `remove snapshot removes from backend and cache`(@TempDir tmpDir: Path) = runBlocking {
val snapshotManager = getSnapshotManager(File(tmpDir.toString()))

val snapshotHandle = AppBackupFileType.Snapshot(repoId, chunkId1)
val file = File(tmpDir.toString(), snapshotHandle.name)
file.createNewFile()
assertTrue(file.isFile)

coEvery { backend.remove(snapshotHandle) } just Runs

snapshotManager.removeSnapshot(snapshotHandle)

assertFalse(file.exists())
coVerify { backend.remove(snapshotHandle) }
}

private fun getSnapshotManager(tmpFolder: File, loader: Loader = this.loader): SnapshotManager {
return SnapshotManager(tmpFolder, crypto, loader, backendManager)
}
}

0 comments on commit 2802ca6

Please sign in to comment.