From d89989153fd6aee713141014a4251a86da6183e8 Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Thu, 4 Apr 2024 17:48:09 -0300 Subject: [PATCH] Ensure root folder exists when using storage We use the same root folder for app and files backup. App backup usually creates the root folder, but if only storage backup is used, it will be missing and needs to be created. --- .../seedvault/plugins/webdav/WebDavStorage.kt | 3 +- .../plugins/webdav/WebDavStoragePlugin.kt | 17 ++++++++++- .../seedvault/storage/WebDavStoragePlugin.kt | 29 ++++++++++++++----- .../ui/recoverycode/RecoveryCodeViewModel.kt | 3 +- .../ui/storage/BackupStorageViewModel.kt | 3 +- .../plugins/webdav/WebDavStoragePluginTest.kt | 8 +++-- .../storage/WebDavStoragePluginTest.kt | 6 ++++ .../seedvault/transport/TransportTest.kt | 7 ++++- .../storagebackuptester/MainViewModel.kt | 9 +++--- .../backup/storage/api/StorageBackup.kt | 10 +++++++ .../backup/storage/api/StoragePlugin.kt | 7 +++++ .../storage/plugin/saf/SafStoragePlugin.kt | 4 +++ 12 files changed, 84 insertions(+), 22 deletions(-) diff --git a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt index 26fdb6965..3027da3bb 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt @@ -38,6 +38,7 @@ const val DIRECTORY_ROOT = ".SeedVaultAndroidBackup" @OptIn(DelicateCoroutinesApi::class) internal abstract class WebDavStorage( webDavConfig: WebDavConfig, + root: String = DIRECTORY_ROOT, ) { companion object { @@ -61,7 +62,7 @@ internal abstract class WebDavStorage( .retryOnConnectionFailure(true) .build() - protected val url = "${webDavConfig.url}/$DIRECTORY_ROOT" + protected val url = "${webDavConfig.url}/$root" @Throws(IOException::class) protected suspend fun getOutputStream(location: HttpUrl): OutputStream { diff --git a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt index 39079a118..b78ad2d26 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt @@ -24,7 +24,8 @@ import kotlin.coroutines.suspendCoroutine internal class WebDavStoragePlugin( context: Context, webDavConfig: WebDavConfig, -) : WebDavStorage(webDavConfig), StoragePlugin { + root: String = DIRECTORY_ROOT, +) : WebDavStorage(webDavConfig, root), StoragePlugin { @Throws(IOException::class) override suspend fun startNewRestoreSet(token: Long) { @@ -39,6 +40,20 @@ internal class WebDavStoragePlugin( override suspend fun initializeDevice() { // TODO does it make sense to delete anything // when [startNewRestoreSet] is always called first? Maybe unify both calls? + val location = url.toHttpUrl() + val davCollection = DavCollection(okHttpClient, location) + + try { + davCollection.head { response -> + debugLog { "Root exists: $response" } + } + } catch (e: NotFoundException) { + val response = davCollection.createFolder() + debugLog { "initializeDevice() = $response" } + } catch (e: Exception) { + if (e is IOException) throw e + else throw IOException(e) + } } @Throws(IOException::class) diff --git a/app/src/main/java/com/stevesoltys/seedvault/storage/WebDavStoragePlugin.kt b/app/src/main/java/com/stevesoltys/seedvault/storage/WebDavStoragePlugin.kt index cbc389d14..3b825d3da 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/storage/WebDavStoragePlugin.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/storage/WebDavStoragePlugin.kt @@ -8,6 +8,7 @@ import at.bitfire.dav4jvm.property.DisplayName import at.bitfire.dav4jvm.property.ResourceType import com.stevesoltys.seedvault.crypto.KeyManager import com.stevesoltys.seedvault.plugins.chunkFolderRegex +import com.stevesoltys.seedvault.plugins.webdav.DIRECTORY_ROOT import com.stevesoltys.seedvault.plugins.webdav.WebDavConfig import com.stevesoltys.seedvault.plugins.webdav.WebDavStorage import okhttp3.HttpUrl.Companion.toHttpUrl @@ -30,7 +31,8 @@ internal class WebDavStoragePlugin( */ androidId: String, webDavConfig: WebDavConfig, -) : WebDavStorage(webDavConfig), StoragePlugin { + root: String = DIRECTORY_ROOT, +) : WebDavStorage(webDavConfig, root), StoragePlugin { /** * The folder name is our user ID plus .sv extension (for SeedVault). @@ -39,6 +41,24 @@ internal class WebDavStoragePlugin( */ private val folder: String = "$androidId.sv" + @Throws(IOException::class) + override suspend fun init() { + val location = url.toHttpUrl() + val davCollection = DavCollection(okHttpClient, location) + + try { + davCollection.head { response -> + debugLog { "Root exists: $response" } + } + } catch (e: NotFoundException) { + val response = davCollection.createFolder() + debugLog { "init() = $response" } + } catch (e: Exception) { + if (e is IOException) throw e + else throw IOException(e) + } + } + @Throws(IOException::class) override suspend fun getAvailableChunkIds(): List { val location = "$url/$folder".toHttpUrl() @@ -211,18 +231,13 @@ internal class WebDavStoragePlugin( val match = snapshotRegex.matchEntire(response.hrefName()) if (match != null) { val timestamp = match.groupValues[1].toLong() - val folderName = - response.href.pathSegments[response.href.pathSegments.size - 2] - val storedSnapshot = StoredSnapshot(folderName, timestamp) + val storedSnapshot = StoredSnapshot(folder, timestamp) snapshots.add(storedSnapshot) } } } } Log.i(TAG, "getCurrentBackupSnapshots took $duration") - } catch (e: NotFoundException) { - debugLog { "Folder not found: $location" } - davCollection.createFolder() } catch (e: Exception) { if (e is IOException) throw e else throw IOException("Error populating chunk folders: ", e) diff --git a/app/src/main/java/com/stevesoltys/seedvault/ui/recoverycode/RecoveryCodeViewModel.kt b/app/src/main/java/com/stevesoltys/seedvault/ui/recoverycode/RecoveryCodeViewModel.kt index 274187c3b..a6fb3effa 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/ui/recoverycode/RecoveryCodeViewModel.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/ui/recoverycode/RecoveryCodeViewModel.kt @@ -103,8 +103,7 @@ internal class RecoveryCodeViewModel( // TODO this code is almost identical to BackupStorageViewModel#onLocationSet(), unify? GlobalScope.launch(Dispatchers.IO) { // remove old storage snapshots and clear cache - storageBackup.deleteAllSnapshots() - storageBackup.clearCache() + storageBackup.init() try { // initialize the new location if (backupManager.isBackupEnabled) backupInitializer.initialize({ }) { diff --git a/app/src/main/java/com/stevesoltys/seedvault/ui/storage/BackupStorageViewModel.kt b/app/src/main/java/com/stevesoltys/seedvault/ui/storage/BackupStorageViewModel.kt index bf6c729bb..b567f0c3a 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/ui/storage/BackupStorageViewModel.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/ui/storage/BackupStorageViewModel.kt @@ -45,8 +45,7 @@ internal class BackupStorageViewModel( // remove old storage snapshots and clear cache // TODO is this needed? It also does create all 255 chunk folders which takes time // pass a flag to getCurrentBackupSnapshots() to not create missing folders? - storageBackup.deleteAllSnapshots() - storageBackup.clearCache() + storageBackup.init() try { // initialize the new location (if backups are enabled) if (backupManager.isBackupEnabled) { diff --git a/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePluginTest.kt b/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePluginTest.kt index 47578ea28..613ecb0dd 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePluginTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePluginTest.kt @@ -34,15 +34,17 @@ internal class WebDavStoragePluginTest : TransportTest() { val token = System.currentTimeMillis() val metadata = getRandomByteArray() + // need to initialize, to have root .SeedVaultAndroidBackup folder + plugin.initializeDevice() + plugin.startNewRestoreSet(token) + // initially, we don't have any backups assertEquals(emptySet(), plugin.getAvailableBackups()?.toSet()) // and no data assertFalse(plugin.hasData(token, FILE_BACKUP_METADATA)) - // start a new restore set, initialize it and write out the metadata file - plugin.startNewRestoreSet(token) - plugin.initializeDevice() + // write out the metadata file plugin.getOutputStream(token, FILE_BACKUP_METADATA).use { it.write(metadata) } diff --git a/app/src/test/java/com/stevesoltys/seedvault/storage/WebDavStoragePluginTest.kt b/app/src/test/java/com/stevesoltys/seedvault/storage/WebDavStoragePluginTest.kt index bcb26612f..1c2671ce9 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/storage/WebDavStoragePluginTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/storage/WebDavStoragePluginTest.kt @@ -28,6 +28,9 @@ internal class WebDavStoragePluginTest : BackupTest() { val chunkId1 = getRandomByteArray(32).toHexString() val chunkBytes1 = getRandomByteArray() + // init to create root folder + plugin.init() + // first we don't have any chunks assertEquals(emptyList(), plugin.getAvailableChunkIds()) @@ -55,6 +58,9 @@ internal class WebDavStoragePluginTest : BackupTest() { fun `test snapshots`() = runBlocking { val snapshotBytes = getRandomByteArray() + // init to create root folder + plugin.init() + // first we don't have any snapshots assertEquals(emptyList(), plugin.getCurrentBackupSnapshots()) assertEquals(emptyList(), plugin.getBackupSnapshotsForRestore()) diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt index b5e58569a..2269093b2 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt @@ -73,6 +73,7 @@ internal abstract class TransportTest { mockkStatic(Log::class) val logTagSlot = slot() val logMsgSlot = slot() + val logExSlot = slot() every { Log.v(any(), any()) } returns 0 every { Log.d(capture(logTagSlot), capture(logMsgSlot)) } answers { println("${logTagSlot.captured} - ${logMsgSlot.captured}") @@ -83,7 +84,11 @@ internal abstract class TransportTest { every { Log.w(any(), ofType(String::class)) } returns 0 every { Log.w(any(), ofType(String::class), any()) } returns 0 every { Log.e(any(), any()) } returns 0 - every { Log.e(any(), any(), any()) } returns 0 + every { Log.e(capture(logTagSlot), capture(logMsgSlot), capture(logExSlot)) } answers { + println("${logTagSlot.captured} - ${logMsgSlot.captured} ${logExSlot.captured}") + logExSlot.captured.printStackTrace() + 0 + } } } diff --git a/storage/demo/src/main/java/de/grobox/storagebackuptester/MainViewModel.kt b/storage/demo/src/main/java/de/grobox/storagebackuptester/MainViewModel.kt index 92a4d5cf4..18159b278 100644 --- a/storage/demo/src/main/java/de/grobox/storagebackuptester/MainViewModel.kt +++ b/storage/demo/src/main/java/de/grobox/storagebackuptester/MainViewModel.kt @@ -32,11 +32,11 @@ import org.calyxos.backup.storage.ui.restore.SnapshotViewModel private val logEmptyState = """ Press the button below to simulate a backup. Your files won't be changed and not uploaded anywhere. This is just to test code for a future real backup. - + Please come back to this app from time to time and run a backup again to see if it correctly identifies files that were added/changed. - + Note that after updating this app, it might need to re-backup all files again. - + Thanks for testing! """.trimIndent() private const val TAG = "MainViewModel" @@ -98,8 +98,7 @@ class MainViewModel(application: Application) : BackupContentViewModel(applicati fun setBackupLocation(uri: Uri?) { if (uri != null) { viewModelScope.launch(Dispatchers.IO) { - storageBackup.deleteAllSnapshots() - storageBackup.clearCache() + storageBackup.init() } } settingsManager.setBackupLocation(uri) diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt index 3c947759d..d5975978c 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt @@ -104,6 +104,16 @@ public class StorageBackup( list.joinToString(", ", limit = 5) } + /** + * Ensures the storage is set-up to receive backups and deletes all snapshots + * (see [deleteAllSnapshots]) as well as clears local cache (see [clearCache]). + */ + public suspend fun init() { + plugin.init() + deleteAllSnapshots() + clearCache() + } + /** * Run this on a new storage location to ensure that there are no old snapshots * (potentially encrypted with an old key) laying around. diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/api/StoragePlugin.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/api/StoragePlugin.kt index e9d474898..a796b8812 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/api/StoragePlugin.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/api/StoragePlugin.kt @@ -13,6 +13,13 @@ import javax.crypto.SecretKey public interface StoragePlugin { + /** + * Prepares the storage location for storing backups. + * Call this before using the [StoragePlugin] for the first time. + */ + @Throws(IOException::class) + public suspend fun init() + /** * Called before starting a backup run to ensure that all cached chunks are still available. * Plugins should use this opportunity diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt index fdb94a91c..7b7905783 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt @@ -75,6 +75,10 @@ public abstract class SafStoragePlugin( return "$timestamp.SeedSnap" } + override suspend fun init() { + // no-op as we are getting [root] created from super class + } + @Throws(IOException::class) override suspend fun getAvailableChunkIds(): List { val folder = folder ?: return emptyList()