diff --git a/Android.bp b/Android.bp index db349e264..0796923f1 100644 --- a/Android.bp +++ b/Android.bp @@ -32,6 +32,9 @@ android_app { "com.google.android.material_material", "kotlinx-coroutines-android", "kotlinx-coroutines-core", + // app backup related libs + "seedvault-lib-kotlin-logging-jvm", + "seedvault-lib-chunker" "seedvault-lib-zstd-jni", // our own gradle module libs "seedvault-lib-core", diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 8af4a16a2..6de1dafca 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -157,6 +157,8 @@ dependencies { implementation(libs.google.protobuf.javalite) implementation(libs.google.tink.android) + implementation(libs.kotlin.logging) + implementation(libs.squareup.okio) /** * Storage Dependencies @@ -175,6 +177,7 @@ dependencies { implementation(fileTree("${rootProject.rootDir}/libs/koin-android").include("*.jar")) implementation(fileTree("${rootProject.rootDir}/libs/koin-android").include("*.aar")) + implementation(fileTree("${rootProject.rootDir}/libs").include("seedvault-chunker-0.1.jar")) implementation(fileTree("${rootProject.rootDir}/libs").include("zstd-jni-1.5.6-5.aar")) implementation(fileTree("${rootProject.rootDir}/libs").include("kotlin-bip39-jvm-1.0.6.jar")) @@ -188,6 +191,7 @@ dependencies { // anything less than 'implementation' fails tests run with gradlew testImplementation(aospLibs) testImplementation("androidx.test.ext:junit:1.1.5") + testImplementation("org.slf4j:slf4j-simple:2.0.3") testImplementation("org.robolectric:robolectric:4.12.2") testImplementation("org.hamcrest:hamcrest:2.2") testImplementation("org.junit.jupiter:junit-jupiter-api:${libs.versions.junit5.get()}") diff --git a/app/src/androidTest/java/com/stevesoltys/seedvault/backend/saf/SafBackendTest.kt b/app/src/androidTest/java/com/stevesoltys/seedvault/backend/saf/SafBackendTest.kt index 6e27738fc..9a2adf88e 100644 --- a/app/src/androidTest/java/com/stevesoltys/seedvault/backend/saf/SafBackendTest.kt +++ b/app/src/androidTest/java/com/stevesoltys/seedvault/backend/saf/SafBackendTest.kt @@ -13,7 +13,6 @@ import kotlinx.coroutines.runBlocking import org.calyxos.seedvault.core.backends.Backend import org.calyxos.seedvault.core.backends.BackendTest import org.calyxos.seedvault.core.backends.saf.SafBackend -import org.calyxos.seedvault.core.backends.saf.SafProperties import org.junit.Test import org.junit.runner.RunWith import org.koin.core.component.KoinComponent @@ -25,14 +24,7 @@ class SafBackendTest : BackendTest(), KoinComponent { private val context = InstrumentationRegistry.getInstrumentation().targetContext private val settingsManager by inject() - private val safStorage = settingsManager.getSafProperties() ?: error("No SAF storage") - private val safProperties = SafProperties( - config = safStorage.config, - name = safStorage.name, - isUsb = safStorage.isUsb, - requiresNetwork = safStorage.requiresNetwork, - rootId = safStorage.rootId, - ) + private val safProperties = settingsManager.getSafProperties() ?: error("No SAF storage") override val backend: Backend = SafBackend(context, safProperties, ".SeedvaultTest") @Test diff --git a/app/src/androidTest/java/com/stevesoltys/seedvault/worker/IconManagerTest.kt b/app/src/androidTest/java/com/stevesoltys/seedvault/worker/IconManagerTest.kt new file mode 100644 index 000000000..3bf0f351e --- /dev/null +++ b/app/src/androidTest/java/com/stevesoltys/seedvault/worker/IconManagerTest.kt @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.worker + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.filters.MediumTest +import androidx.test.platform.app.InstrumentationRegistry +import com.google.protobuf.ByteString +import com.stevesoltys.seedvault.BackupStateManager +import com.stevesoltys.seedvault.proto.Snapshot +import com.stevesoltys.seedvault.transport.backup.BackupData +import com.stevesoltys.seedvault.transport.backup.BackupReceiver +import com.stevesoltys.seedvault.transport.backup.PackageService +import com.stevesoltys.seedvault.transport.backup.SnapshotCreatorFactory +import com.stevesoltys.seedvault.transport.restore.Loader +import io.mockk.Runs +import io.mockk.coEvery +import io.mockk.every +import io.mockk.just +import io.mockk.mockk +import io.mockk.slot +import junit.framework.TestCase.assertTrue +import kotlinx.coroutines.runBlocking +import org.calyxos.seedvault.core.backends.AppBackupFileType +import org.calyxos.seedvault.core.toHexString +import org.junit.Assert.assertArrayEquals +import org.junit.Test +import org.junit.runner.RunWith +import org.koin.core.component.KoinComponent +import org.koin.core.component.inject +import java.io.ByteArrayInputStream +import java.io.InputStream +import kotlin.random.Random + +@RunWith(AndroidJUnit4::class) +@MediumTest +class IconManagerTest : KoinComponent { + + private val context = InstrumentationRegistry.getInstrumentation().targetContext + private val packageService by inject() + private val backupReceiver = mockk() + private val loader = mockk() + private val backupStateManager = mockk() + private val snapshotCreatorFactory by inject() + private val snapshotCreator = snapshotCreatorFactory.createSnapshotCreator() + + private val iconManager = IconManager( + context = context, + packageService = packageService, + backupReceiver = backupReceiver, + loader = loader, + backupStateManager = backupStateManager, + ) + + init { + every { backupStateManager.snapshotCreator } returns snapshotCreator + } + + @Test + fun `test upload and then download`(): Unit = runBlocking { + // prepare output data + val output = slot() + val chunkId = Random.nextBytes(32).toHexString() + val chunkList = listOf(chunkId) + val blobId = Random.nextBytes(32).toHexString() + val blob = Snapshot.Blob.newBuilder().setId(ByteString.fromHex(blobId)).build() + + // upload icons and capture plaintext bytes + coEvery { backupReceiver.addBytes(capture(output)) } just Runs + coEvery { backupReceiver.finalize() } returns BackupData(chunkList, mapOf(chunkId to blob)) + iconManager.uploadIcons() + assertTrue(output.captured.isNotEmpty()) + + // get snapshot and assert it has icon chunks + val snapshot = snapshotCreator.finalizeSnapshot() + assertTrue(snapshot.iconChunkIdsCount > 0) + + // prepare data for downloading icons + val repoId = Random.nextBytes(32).toHexString() + val inputStream = ByteArrayInputStream(output.captured) + coEvery { + loader.loadFile(AppBackupFileType.Blob(repoId, blobId), captureLambda()) + } answers { + lambda<(InputStream) -> Unit>().captured.invoke(inputStream) + } + + // download icons and ensure we had an icon for at least one app + val iconSet = iconManager.downloadIcons(repoId, snapshot) + assertTrue(iconSet.isNotEmpty()) + } + + @Test + fun `test upload produces deterministic output`(): Unit = runBlocking { + val output1 = slot() + val output2 = slot() + + coEvery { backupReceiver.addBytes(capture(output1)) } just Runs + coEvery { backupReceiver.finalize() } returns BackupData(emptyList(), emptyMap()) + iconManager.uploadIcons() + assertTrue(output1.captured.isNotEmpty()) + + coEvery { backupReceiver.addBytes(capture(output2)) } just Runs + coEvery { backupReceiver.finalize() } returns BackupData(emptyList(), emptyMap()) + iconManager.uploadIcons() + assertTrue(output2.captured.isNotEmpty()) + + assertArrayEquals(output1.captured, output2.captured) + } + +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/header/Header.kt b/app/src/main/java/com/stevesoltys/seedvault/header/Header.kt index e95640bc8..9489868a9 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/header/Header.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/header/Header.kt @@ -10,7 +10,7 @@ import com.stevesoltys.seedvault.crypto.TYPE_BACKUP_FULL import com.stevesoltys.seedvault.crypto.TYPE_BACKUP_KV import java.nio.ByteBuffer -internal const val VERSION: Byte = 1 +internal const val VERSION: Byte = 2 internal const val MAX_PACKAGE_LENGTH_SIZE = 255 internal const val MAX_KEY_LENGTH_SIZE = MAX_PACKAGE_LENGTH_SIZE internal const val MAX_VERSION_HEADER_SIZE = diff --git a/app/src/main/java/com/stevesoltys/seedvault/metadata/Metadata.kt b/app/src/main/java/com/stevesoltys/seedvault/metadata/Metadata.kt index 362275acd..fd5fd6b94 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/metadata/Metadata.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/metadata/Metadata.kt @@ -8,8 +8,12 @@ package com.stevesoltys.seedvault.metadata import android.content.pm.ApplicationInfo.FLAG_STOPPED import android.os.Build import com.stevesoltys.seedvault.crypto.TYPE_METADATA +import com.stevesoltys.seedvault.encodeBase64 import com.stevesoltys.seedvault.header.VERSION import com.stevesoltys.seedvault.metadata.PackageState.UNKNOWN_ERROR +import com.stevesoltys.seedvault.proto.Snapshot +import com.stevesoltys.seedvault.transport.backup.hexFromProto +import com.stevesoltys.seedvault.worker.BASE_SPLIT import org.calyxos.backup.storage.crypto.StreamCrypto.toByteArray import java.nio.ByteBuffer @@ -91,12 +95,51 @@ data class PackageMetadata( internal val version: Long? = null, internal val installer: String? = null, internal val splits: List? = null, + internal val chunkIds: List? = null, // used for v2 internal val sha256: String? = null, internal val signatures: List? = null, ) { + + companion object { + fun fromSnapshot(app: Snapshot.App) = PackageMetadata( + time = app.time, + state = if (app.state.isBlank()) UNKNOWN_ERROR else PackageState.valueOf(app.state), + backupType = when (app.type) { + Snapshot.BackupType.FULL -> BackupType.FULL + Snapshot.BackupType.KV -> BackupType.KV + else -> null + }, + name = app.name, + system = app.system, + isLaunchableSystemApp = app.launchableSystemApp, + version = app.apk.versionCode, + installer = app.apk.installer, + splits = app.apk.splitsList.filter { it.name != BASE_SPLIT }.map { + ApkSplit( + name = it.name, + size = null, + sha256 = "", + chunkIds = if (it.chunkIdsCount == 0) null else it.chunkIdsList.hexFromProto() + ) + }, + chunkIds = run { + val baseChunk = app.apk.splitsList.find { it.name == BASE_SPLIT } + if (baseChunk == null || baseChunk.chunkIdsCount == 0) { + null + } else { + baseChunk.chunkIdsList.hexFromProto() + } + }, + sha256 = null, + signatures = app.apk.signaturesList.map { it.toByteArray().encodeBase64() }, + ) + } + val isInternalSystem: Boolean = system && !isLaunchableSystemApp fun hasApk(): Boolean { - return version != null && sha256 != null && signatures != null + return version != null && + (sha256 != null || chunkIds?.isNotEmpty() == true) && // v2 doesn't use sha256 here + signatures != null } } @@ -104,6 +147,7 @@ data class ApkSplit( val name: String, val size: Long?, val sha256: String, + val chunkIds: List? = null, // used for v2 // There's also a revisionCode, but it doesn't seem to be used just yet ) diff --git a/app/src/main/java/com/stevesoltys/seedvault/restore/AppSelectionManager.kt b/app/src/main/java/com/stevesoltys/seedvault/restore/AppSelectionManager.kt index 22a0b4ee5..ab5ec1909 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/restore/AppSelectionManager.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/restore/AppSelectionManager.kt @@ -12,9 +12,10 @@ import androidx.lifecycle.asLiveData import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER import com.stevesoltys.seedvault.NO_DATA_END_SENTINEL import com.stevesoltys.seedvault.R +import com.stevesoltys.seedvault.backend.BackendManager import com.stevesoltys.seedvault.metadata.PackageMetadata import com.stevesoltys.seedvault.metadata.PackageMetadataMap -import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.proto.Snapshot import com.stevesoltys.seedvault.ui.PACKAGE_NAME_SYSTEM import com.stevesoltys.seedvault.ui.systemData import com.stevesoltys.seedvault.worker.IconManager @@ -24,7 +25,6 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch -import org.calyxos.seedvault.core.backends.LegacyAppBackupFile import java.util.Locale internal class SelectedAppsState( @@ -37,7 +37,7 @@ private val TAG = AppSelectionManager::class.simpleName internal class AppSelectionManager( private val context: Context, - private val backendManager: BackendManager, + private val backendManager: BackendManager, // TODO remove private val iconManager: IconManager, private val coroutineScope: CoroutineScope, private val workDispatcher: CoroutineDispatcher = Dispatchers.IO, @@ -88,12 +88,11 @@ internal class AppSelectionManager( SelectedAppsState(apps = items, allSelected = isSetupWizard, iconsLoaded = false) // download icons coroutineScope.launch(workDispatcher) { - val backend = backendManager.backend - val token = restorableBackup.token val packagesWithIcons = try { - backend.load(LegacyAppBackupFile.IconsFile(token)).use { - iconManager.downloadIcons(restorableBackup.version, token, it) - } + // TODO get real repoId + val repoId = "3f1f3d9da0fd5a509196cc96b75c668172592fcb5c20b9159f398da2b6149cc1" + // TODO get real snapshot + iconManager.downloadIcons(repoId, Snapshot.newBuilder().build()) } catch (e: Exception) { Log.e(TAG, "Error loading icons:", e) emptySet() diff --git a/app/src/main/java/com/stevesoltys/seedvault/restore/RestorableBackup.kt b/app/src/main/java/com/stevesoltys/seedvault/restore/RestorableBackup.kt index a895f91a3..f47c7a816 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/restore/RestorableBackup.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/restore/RestorableBackup.kt @@ -7,8 +7,13 @@ package com.stevesoltys.seedvault.restore import com.stevesoltys.seedvault.metadata.BackupMetadata import com.stevesoltys.seedvault.metadata.PackageMetadataMap +import com.stevesoltys.seedvault.proto.Snapshot -data class RestorableBackup(val backupMetadata: BackupMetadata) { +data class RestorableBackup( + val backupMetadata: BackupMetadata, + val repoId: String? = null, + val snapshot: Snapshot? = null, +) { val name: String get() = backupMetadata.deviceName diff --git a/app/src/main/java/com/stevesoltys/seedvault/restore/install/ApkRestore.kt b/app/src/main/java/com/stevesoltys/seedvault/restore/install/ApkRestore.kt index 3fe0f700b..347c8f380 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/restore/install/ApkRestore.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/restore/install/ApkRestore.kt @@ -10,31 +10,38 @@ import android.content.Context import android.content.pm.PackageManager import android.content.pm.PackageManager.GET_SIGNATURES import android.content.pm.PackageManager.GET_SIGNING_CERTIFICATES +import android.content.pm.SigningInfo import android.util.Log import com.stevesoltys.seedvault.BackupStateManager import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.backend.LegacyStoragePlugin import com.stevesoltys.seedvault.crypto.Crypto +import com.stevesoltys.seedvault.encodeBase64 import com.stevesoltys.seedvault.metadata.ApkSplit import com.stevesoltys.seedvault.metadata.PackageMetadata -import com.stevesoltys.seedvault.backend.BackendManager -import com.stevesoltys.seedvault.backend.LegacyStoragePlugin import com.stevesoltys.seedvault.restore.RestorableBackup import com.stevesoltys.seedvault.restore.install.ApkInstallState.FAILED import com.stevesoltys.seedvault.restore.install.ApkInstallState.FAILED_SYSTEM_APP import com.stevesoltys.seedvault.restore.install.ApkInstallState.IN_PROGRESS import com.stevesoltys.seedvault.restore.install.ApkInstallState.QUEUED import com.stevesoltys.seedvault.restore.install.ApkInstallState.SUCCEEDED +import com.stevesoltys.seedvault.transport.backup.hexFromProto import com.stevesoltys.seedvault.transport.backup.isSystemApp -import com.stevesoltys.seedvault.worker.copyStreamsAndGetHash -import com.stevesoltys.seedvault.worker.getSignatures +import com.stevesoltys.seedvault.transport.restore.Loader +import com.stevesoltys.seedvault.worker.hashSignature import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.update +import org.calyxos.seedvault.core.backends.AppBackupFileType import org.calyxos.seedvault.core.backends.Backend import org.calyxos.seedvault.core.backends.LegacyAppBackupFile import java.io.File import java.io.IOException +import java.io.InputStream +import java.io.OutputStream +import java.security.MessageDigest import java.util.Locale private val TAG = ApkRestore::class.java.simpleName @@ -44,6 +51,7 @@ internal class ApkRestore( private val backupManager: IBackupManager, private val backupStateManager: BackupStateManager, private val backendManager: BackendManager, + private val loader: Loader, @Suppress("Deprecation") private val legacyStoragePlugin: LegacyStoragePlugin, private val crypto: Crypto, @@ -162,10 +170,10 @@ internal class ApkRestore( } // cache the APK and get its hash - val (cachedApk, sha256) = cacheApk(backup.version, backup.token, backup.salt, packageName) + val (cachedApk, sha256) = cacheApk(backup, packageName, metadata.chunkIds) - // check APK's SHA-256 hash - if (metadata.sha256 != sha256) throw SecurityException( + // check APK's SHA-256 hash for backup versions before 2 + if (backup.version < 2 && metadata.sha256 != sha256) throw SecurityException( "Package $packageName has sha256 '$sha256', but '${metadata.sha256}' expected." ) @@ -256,10 +264,9 @@ internal class ApkRestore( } splits.forEach { apkSplit -> // cache and check all splits val suffix = if (backup.version == 0.toByte()) "_${apkSplit.sha256}" else apkSplit.name - val salt = backup.salt - val (file, sha256) = cacheApk(backup.version, backup.token, salt, packageName, suffix) - // check APK split's SHA-256 hash - if (apkSplit.sha256 != sha256) throw SecurityException( + val (file, sha256) = cacheApk(backup, packageName, apkSplit.chunkIds, suffix) + // check APK split's SHA-256 hash for backup versions before 2 + if (backup.version < 2 && apkSplit.sha256 != sha256) throw SecurityException( "$packageName:${apkSplit.name} has sha256 '$sha256'," + " but '${apkSplit.sha256}' expected." ) @@ -276,20 +283,32 @@ internal class ApkRestore( */ @Throws(IOException::class) private suspend fun cacheApk( - version: Byte, - token: Long, - salt: String, + backup: RestorableBackup, packageName: String, + chunkIds: List?, suffix: String = "", ): Pair { // create a cache file to write the APK into val cachedApk = File.createTempFile(packageName + suffix, ".apk", context.cacheDir) // copy APK to cache file and calculate SHA-256 hash while we are at it - val inputStream = if (version == 0.toByte()) { - legacyStoragePlugin.getApkInputStream(token, packageName, suffix) - } else { - val name = crypto.getNameForApk(salt, packageName, suffix) - backend.load(LegacyAppBackupFile.Blob(token, name)) + val inputStream = when (backup.version) { + 0.toByte() -> { + legacyStoragePlugin.getApkInputStream(backup.token, packageName, suffix) + } + 1.toByte() -> { + val name = crypto.getNameForApk(backup.salt, packageName, suffix) + backend.load(LegacyAppBackupFile.Blob(backup.token, name)) + } + else -> { + val repoId = backup.repoId ?: error("No repoId for v2 backup") + val snapshot = backup.snapshot ?: error("No snapshot for v2 backup") + val handles = chunkIds?.map { chunkId -> + val blobId = snapshot.blobsMap[chunkId]?.id?.hexFromProto() + ?: error("Blob for $chunkId missing from snapshot ${snapshot.token}") + AppBackupFileType.Blob(repoId, blobId) + } ?: error("No chunkIds for $packageName-$suffix") + loader.loadFiles(handles) + } } val sha256 = copyStreamsAndGetHash(inputStream, cachedApk.outputStream()) return Pair(cachedApk, sha256) @@ -337,3 +356,45 @@ internal class ApkRestore( } } } + +/** + * Copy the APK from the given [InputStream] to the given [OutputStream] + * and calculate the SHA-256 hash while at it. + * + * Both streams will be closed when the method returns. + * + * @return the APK's SHA-256 hash in Base64 format. + */ +@Throws(IOException::class) +fun copyStreamsAndGetHash(inputStream: InputStream, outputStream: OutputStream): String { + val messageDigest = MessageDigest.getInstance("SHA-256") + outputStream.use { oStream -> + inputStream.use { inputStream -> + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + var bytes = inputStream.read(buffer) + while (bytes >= 0) { + oStream.write(buffer, 0, bytes) + messageDigest.update(buffer, 0, bytes) + bytes = inputStream.read(buffer) + } + } + } + return messageDigest.digest().encodeBase64() +} + +/** + * Returns a list of Base64 encoded SHA-256 signature hashes. + */ +fun SigningInfo?.getSignatures(): List { + return if (this == null) { + emptyList() + } else if (hasMultipleSigners()) { + apkContentsSigners.map { signature -> + hashSignature(signature).encodeBase64() + } + } else { + signingCertificateHistory.map { signature -> + hashSignature(signature).encodeBase64() + } + } +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/restore/install/InstallModule.kt b/app/src/main/java/com/stevesoltys/seedvault/restore/install/InstallModule.kt index 8fbeaaaea..7b676c1a5 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/restore/install/InstallModule.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/restore/install/InstallModule.kt @@ -14,7 +14,7 @@ val installModule = module { factory { DeviceInfo(androidContext()) } factory { ApkSplitCompatibilityChecker(get()) } factory { - ApkRestore(androidContext(), get(), get(), get(), get(), get(), get(), get()) { + ApkRestore(androidContext(), get(), get(), get(), get(), get(), get(), get(), get()) { androidContext().getSystemService(UserManager::class.java)!!.isAllowedToInstallApks() } } diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/SnapshotManager.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/SnapshotManager.kt new file mode 100644 index 000000000..1864e1e1b --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/SnapshotManager.kt @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport + +import com.github.luben.zstd.ZstdOutputStream +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.crypto.Crypto +import com.stevesoltys.seedvault.header.VERSION +import com.stevesoltys.seedvault.proto.Snapshot +import com.stevesoltys.seedvault.transport.restore.Loader +import io.github.oshai.kotlinlogging.KotlinLogging +import okio.Buffer +import okio.buffer +import okio.sink +import org.calyxos.seedvault.core.backends.AppBackupFileType +import org.calyxos.seedvault.core.backends.TopLevelFolder + +internal class SnapshotManager( + private val crypto: Crypto, + private val loader: Loader, + private val backendManager: BackendManager, +) { + + private val log = KotlinLogging.logger {} + + /** + * The latest [Snapshot]. May be stale if [loadSnapshots] has not returned + * or wasn't called since new snapshots have been created. + */ + var latestSnapshot: Snapshot? = null + private set + + suspend fun loadSnapshots(callback: (Snapshot) -> Unit) { + log.info { "Loading snapshots..." } + val handles = mutableListOf() + backendManager.backend.list( + topLevelFolder = TopLevelFolder(crypto.repoId), + AppBackupFileType.Snapshot::class, + ) { fileInfo -> + fileInfo.fileHandle as AppBackupFileType.Snapshot + handles.add(fileInfo.fileHandle as AppBackupFileType.Snapshot) + } + handles.forEach { fileHandle -> + val snapshot = onSnapshotFound(fileHandle) + callback(snapshot) + } + } + + private suspend fun onSnapshotFound(snapshotHandle: AppBackupFileType.Snapshot): Snapshot { + // TODO set up local snapshot cache, so we don't need to download those all the time + val snapshot = loader.loadFile(snapshotHandle).use { + Snapshot.parseFrom(it) + } + // update latest snapshot if this one is more recent + if (snapshot.token > (latestSnapshot?.token ?: 0)) latestSnapshot = snapshot + return snapshot + } + + suspend fun saveSnapshot(snapshot: Snapshot) { + val buffer = Buffer() + val bufferStream = buffer.outputStream() + bufferStream.write(VERSION.toInt()) + crypto.newEncryptingStream(bufferStream, crypto.getAdForVersion()).use { cryptoStream -> + ZstdOutputStream(cryptoStream).use { zstdOutputStream -> + snapshot.writeTo(zstdOutputStream) + } + } + val sha256ByteString = buffer.sha256() + val handle = AppBackupFileType.Snapshot(crypto.repoId, sha256ByteString.hex()) + // TODO exception handling + backendManager.backend.save(handle).use { outputStream -> + outputStream.sink().buffer().writeAll(buffer) + } + } + +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/AppBackupManager.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/AppBackupManager.kt new file mode 100644 index 000000000..7949403ec --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/AppBackupManager.kt @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.backup + +import com.stevesoltys.seedvault.transport.SnapshotManager +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.delay + +internal class AppBackupManager( + private val blobsCache: BlobsCache, + private val snapshotManager: SnapshotManager, + private val snapshotCreatorFactory: SnapshotCreatorFactory, +) { + + private val log = KotlinLogging.logger {} + var snapshotCreator: SnapshotCreator? = null + private set + + suspend fun beforeBackup() { + log.info { "Before backup" } + snapshotCreator = snapshotCreatorFactory.createSnapshotCreator() + blobsCache.populateCache() + } + + suspend fun afterBackupFinished() { + log.info { "After backup finished" } + blobsCache.clear() + val snapshot = snapshotCreator?.finalizeSnapshot() ?: error("Had no snapshotCreator") + keepTrying { + snapshotManager.saveSnapshot(snapshot) + } + snapshotCreator = null + } + + private suspend fun keepTrying(n: Int = 3, block: suspend () -> Unit) { + for (i in 1..n) { + try { + block() + return + } catch (e: Exception) { + if (i == n) throw e + log.error(e) { "Error (#$i), we'll keep trying" } + delay(1000) + } + } + } + +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt index bc18e0cbd..206bee0db 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt @@ -5,11 +5,17 @@ package com.stevesoltys.seedvault.transport.backup +import com.stevesoltys.seedvault.transport.SnapshotManager import org.koin.android.ext.koin.androidContext import org.koin.dsl.module val backupModule = module { single { BackupInitializer(get()) } + single { BackupReceiver(get(), get(), get()) } + single { BlobsCache(get(), get(), get()) } + single { BlobCreator(get(), get()) } + single { SnapshotManager(get(), get(), get()) } + single { SnapshotCreatorFactory(androidContext(), get(), get(), get()) } single { InputFactory() } single { PackageService( diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupReceiver.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupReceiver.kt new file mode 100644 index 000000000..3b55418ad --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupReceiver.kt @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.backup + +import com.stevesoltys.seedvault.crypto.Crypto +import com.stevesoltys.seedvault.proto.Snapshot.Blob +import org.calyxos.seedvault.chunker.Chunk +import org.calyxos.seedvault.chunker.Chunker +import org.calyxos.seedvault.chunker.GearTableCreator +import org.calyxos.seedvault.core.toHexString +import java.io.InputStream + +data class BackupData( + val chunks: List, + val chunkMap: Map, +) + +internal class BackupReceiver( + private val blobsCache: BlobsCache, + private val blobCreator: BlobCreator, + private val crypto: Crypto, + private val chunker: Chunker = Chunker( + minSize = 1536 * 1024, // 1.5 MB + avgSize = 3 * 1024 * 1024, // 3.0 MB + maxSize = 7680 * 1024, // 7.5 MB + normalization = 1, + gearTable = GearTableCreator.create(crypto.gearTableKey), + hashFunction = { bytes -> + crypto.sha256(bytes).toHexString() + }, + ) +) { + + private val chunks = mutableListOf() + private val chunkMap = mutableMapOf() + + suspend fun addBytes(bytes: ByteArray) { + chunker.addBytes(bytes).forEach { chunk -> + onNewChunk(chunk) + } + } + + suspend fun readFromStream(inputStream: InputStream) { + try { + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + var bytes = inputStream.read(buffer) + while (bytes >= 0) { + if (bytes == buffer.size) { + addBytes(buffer) + } else { + addBytes(buffer.copyOfRange(0, bytes)) + } + bytes = inputStream.read(buffer) + } + } catch (e: Exception) { + finalize() + throw e + } + } + + suspend fun finalize(): BackupData { + chunker.finalize().forEach { chunk -> + onNewChunk(chunk) + } + // copy chunks and chunkMap before clearing + val backupData = BackupData(chunks.toList(), chunkMap.toMap()) + chunks.clear() + chunkMap.clear() + return backupData + // TODO add to SnapshotCreator one level up for inclusion in snapshot + } + + private suspend fun onNewChunk(chunk: Chunk) { + chunks.add(chunk.hash) + + val existingBlob = blobsCache.getBlob(chunk.hash) + if (existingBlob == null) { + val blob = blobCreator.createNewBlob(chunk) + chunkMap[chunk.hash] = blob + blobsCache.saveNewBlob(chunk.hash, blob) + } else { + chunkMap[chunk.hash] = existingBlob + } + } + +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobCreator.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobCreator.kt new file mode 100644 index 000000000..15b9f6208 --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobCreator.kt @@ -0,0 +1,48 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.backup + +import com.github.luben.zstd.ZstdOutputStream +import com.google.protobuf.ByteString +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.crypto.Crypto +import com.stevesoltys.seedvault.header.VERSION +import com.stevesoltys.seedvault.proto.Snapshot.Blob +import okio.Buffer +import okio.buffer +import okio.sink +import org.calyxos.seedvault.chunker.Chunk +import org.calyxos.seedvault.core.backends.AppBackupFileType + +internal class BlobCreator( + private val crypto: Crypto, + private val backendManager: BackendManager, +) { + + private val buffer = Buffer() + + suspend fun createNewBlob(chunk: Chunk): Blob { + buffer.clear() + val bufferStream = buffer.outputStream() + bufferStream.write(VERSION.toInt()) + crypto.newEncryptingStream(bufferStream, crypto.getAdForVersion()).use { cryptoStream -> + ZstdOutputStream(cryptoStream).use { zstdOutputStream -> + zstdOutputStream.write(chunk.data) + } + } + val sha256ByteString = buffer.sha256() + val handle = AppBackupFileType.Blob(crypto.repoId, sha256ByteString.hex()) + // TODO exception handling and retries + val size = backendManager.backend.save(handle).use { outputStream -> + outputStream.sink().buffer().writeAll(buffer) + } + return Blob.newBuilder() + .setId(ByteString.copyFrom(sha256ByteString.asByteBuffer())) + .setLength(size.toInt()) + .setUncompressedLength(chunk.length) + .build() + } +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobsCache.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobsCache.kt new file mode 100644 index 000000000..9d7f715e3 --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobsCache.kt @@ -0,0 +1,69 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.backup + +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.crypto.Crypto +import com.stevesoltys.seedvault.proto.Snapshot.Blob +import com.stevesoltys.seedvault.transport.SnapshotManager +import io.github.oshai.kotlinlogging.KotlinLogging +import org.calyxos.seedvault.core.backends.AppBackupFileType +import org.calyxos.seedvault.core.backends.TopLevelFolder + +internal class BlobsCache( + private val crypto: Crypto, + private val backendManager: BackendManager, + private val snapshotManager: SnapshotManager, +) { + + private val log = KotlinLogging.logger {} + private val blobMap = mutableMapOf() + + /** + * This must be called before saving files to the backend to avoid uploading duplicate blobs. + */ + suspend fun populateCache() { + log.info { "Getting all blobs from backend..." } + blobMap.clear() + val blobs = mutableSetOf() + backendManager.backend.list( + topLevelFolder = TopLevelFolder(crypto.repoId), + AppBackupFileType.Blob::class, + ) { fileInfo -> + fileInfo.fileHandle as AppBackupFileType.Blob + // TODO we could save size info here and later check it is as expected + blobs.add(fileInfo.fileHandle.name) + } + snapshotManager.loadSnapshots { snapshot -> + snapshot.blobsMap.forEach { (chunkId, blob) -> + // check if referenced blob still exists on backend + if (blobs.contains(blob.id.hexFromProto())) { + // only add blob to our mapping, if it still exists + blobMap.putIfAbsent(chunkId, blob)?.let { previous -> + if (previous.id != blob.id) log.warn { + "Chunk ID ${chunkId.substring(0..5)} had more than one blob" + } + } + } else log.warn { + "Blob ${blob.id.hexFromProto()} referenced in snapshot ${snapshot.token}" + } + } + } + } + + fun getBlob(hash: String): Blob? = blobMap[hash] + + fun saveNewBlob(chunkId: String, blob: Blob) { + blobMap[chunkId] = blob + // TODO persist this blob locally in case backup gets interrupted + } + + fun clear() { + log.info { "Clearing cache..." } + blobMap.clear() + } + +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/SnapshotCreator.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/SnapshotCreator.kt new file mode 100644 index 000000000..adbf64eb1 --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/SnapshotCreator.kt @@ -0,0 +1,132 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.backup + +import android.annotation.SuppressLint +import android.content.Context +import android.content.pm.PackageInfo +import android.content.pm.PackageManager.PERMISSION_GRANTED +import android.os.Build +import android.os.UserManager +import android.provider.Settings +import android.provider.Settings.Secure.ANDROID_ID +import com.google.protobuf.ByteString +import com.stevesoltys.seedvault.Clock +import com.stevesoltys.seedvault.metadata.BackupType +import com.stevesoltys.seedvault.metadata.PackageState.APK_AND_DATA +import com.stevesoltys.seedvault.proto.Snapshot +import com.stevesoltys.seedvault.proto.Snapshot.Apk +import com.stevesoltys.seedvault.proto.Snapshot.App +import com.stevesoltys.seedvault.proto.Snapshot.Blob +import com.stevesoltys.seedvault.settings.SettingsManager +import org.calyxos.seedvault.core.toHexString + +internal class SnapshotCreatorFactory( + private val context: Context, + private val clock: Clock, + private val packageService: PackageService, + private val settingsManager: SettingsManager, +) { + fun createSnapshotCreator() = SnapshotCreator(context, clock, packageService, settingsManager) +} + +internal class SnapshotCreator( + private val context: Context, + private val clock: Clock, + private val packageService: PackageService, + private val settingsManager: SettingsManager, +) { + + private val snapshotBuilder = Snapshot.newBuilder() + .setToken(clock.time()) + private val appBuilderMap = mutableMapOf() + private val blobsMap = mutableMapOf() + + private val launchableSystemApps by lazy { + packageService.launchableSystemApps.map { it.activityInfo.packageName }.toSet() + } + + fun onApkBackedUp( + packageName: String, + apk: Apk, + chunkMap: Map, + ) { + val appBuilder = appBuilderMap.getOrPut(packageName) { + App.newBuilder() + } + appBuilder.setApk(apk) + blobsMap.putAll(chunkMap) + } + + fun onPackageBackedUp( + packageInfo: PackageInfo, + type: BackupType, + backupData: BackupData, + ) { + val packageName = packageInfo.packageName + val builder = appBuilderMap.getOrPut(packageName) { + App.newBuilder() + } + val isSystemApp = packageInfo.isSystemApp() + val chunkIds = backupData.chunks.forProto() + blobsMap.putAll(backupData.chunkMap) + builder + .setTime(clock.time()) + .setState(APK_AND_DATA.name) + .setType(type.forSnapshot()) + .setName(packageInfo.applicationInfo?.loadLabel(context.packageManager)?.toString()) + .setSystem(isSystemApp) + .setLaunchableSystemApp(isSystemApp && launchableSystemApps.contains(packageName)) + .addAllChunkIds(chunkIds) + } + + fun onIconsBackedUp(backupData: BackupData) { + snapshotBuilder.addAllIconChunkIds(backupData.chunks.forProto()) + blobsMap.putAll(backupData.chunkMap) + } + + fun finalizeSnapshot(): Snapshot { + val userName = getUserName() + val deviceName = if (userName == null) { + "${Build.MANUFACTURER} ${Build.MODEL}" + } else { + "${Build.MANUFACTURER} ${Build.MODEL} - $userName" + } + + @SuppressLint("HardwareIds") + val androidId = Settings.Secure.getString(context.contentResolver, ANDROID_ID) + val snapshot = snapshotBuilder + .setName(deviceName) + .setAndroidId(androidId) + .setSdkInt(Build.VERSION.SDK_INT) + .setAndroidIncremental(Build.VERSION.INCREMENTAL) + .setD2D(settingsManager.d2dBackupsEnabled()) + .putAllApps(appBuilderMap.mapValues { it.value.build() }) + .putAllBlobs(blobsMap) + .build() + appBuilderMap.clear() + snapshotBuilder.clear() + return snapshot + } + + private fun getUserName(): String? { + val perm = "android.permission.QUERY_USERS" + return if (context.checkSelfPermission(perm) == PERMISSION_GRANTED) { + val userManager = context.getSystemService(UserManager::class.java) ?: return null + userManager.userName + } else null + } + + private fun BackupType.forSnapshot(): Snapshot.BackupType = when (this) { + BackupType.KV -> Snapshot.BackupType.KV + BackupType.FULL -> Snapshot.BackupType.FULL + } + +} + +fun Iterable.forProto() = map { ByteString.fromHex(it) } +fun Iterable.hexFromProto() = map { it.toByteArray().toHexString() } +fun ByteString.hexFromProto() = toByteArray().toHexString() diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/restore/Loader.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/restore/Loader.kt new file mode 100644 index 000000000..8d231bb66 --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/restore/Loader.kt @@ -0,0 +1,71 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.restore + +import com.github.luben.zstd.ZstdInputStream +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.crypto.Crypto +import com.stevesoltys.seedvault.header.UnsupportedVersionException +import com.stevesoltys.seedvault.header.VERSION +import kotlinx.coroutines.runBlocking +import org.calyxos.seedvault.core.backends.AppBackupFileType +import org.calyxos.seedvault.core.toHexString +import java.io.ByteArrayInputStream +import java.io.InputStream +import java.io.SequenceInputStream +import java.security.GeneralSecurityException +import java.util.Enumeration + +internal class Loader( + private val crypto: Crypto, + private val backendManager: BackendManager, +) { + + /** + * The responsibility with closing the returned stream lies with the caller. + */ + suspend fun loadFile(handle: AppBackupFileType): InputStream { + // We load the entire ciphertext into memory, + // so we can check the SHA-256 hash before decrypting and parsing the data. + val cipherText = backendManager.backend.load(handle).use { inputStream -> + inputStream.readAllBytes() + } + // check SHA-256 hash first thing + val sha256 = crypto.sha256(cipherText).toHexString() + val expectedHash = when (handle) { + is AppBackupFileType.Snapshot -> handle.hash + is AppBackupFileType.Blob -> handle.name + } + if (sha256 != expectedHash) { + throw GeneralSecurityException("Snapshot had wrong SHA-256 hash: $handle") + } + // check that we can handle the version of that snapshot + val version = cipherText[0] + if (version <= 1) throw GeneralSecurityException("Unexpected version: $version") + if (version > VERSION) throw UnsupportedVersionException(version) + // get associated data for version, used for authenticated decryption + val ad = crypto.getAdForVersion(version) + // skip first version byte when creating cipherText stream + val inputStream = ByteArrayInputStream(cipherText, 1, cipherText.size - 1) + // decrypt and decompress cipherText stream and parse snapshot + return ZstdInputStream(crypto.newDecryptingStream(inputStream, ad)) + } + + suspend fun loadFiles(handles: List): InputStream { + val enumeration: Enumeration = object : Enumeration { + val iterator = handles.iterator() + + override fun hasMoreElements(): Boolean { + return iterator.hasNext() + } + + override fun nextElement(): InputStream { + return runBlocking { loadFile(iterator.next()) } + } + } + return SequenceInputStream(enumeration) + } +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/restore/RestoreModule.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/restore/RestoreModule.kt index 869e1b0ca..fd1d835d6 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/transport/restore/RestoreModule.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/restore/RestoreModule.kt @@ -10,6 +10,7 @@ import org.koin.dsl.module val restoreModule = module { single { OutputFactory() } + single { Loader(get(), get()) } single { KVRestore(get(), get(), get(), get(), get(), get()) } single { FullRestore(get(), get(), get(), get(), get()) } single { diff --git a/app/src/main/java/com/stevesoltys/seedvault/ui/notification/NotificationBackupObserver.kt b/app/src/main/java/com/stevesoltys/seedvault/ui/notification/NotificationBackupObserver.kt index d45dca919..1c93f9f7b 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/ui/notification/NotificationBackupObserver.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/ui/notification/NotificationBackupObserver.kt @@ -19,8 +19,10 @@ import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER import com.stevesoltys.seedvault.R import com.stevesoltys.seedvault.metadata.MetadataManager import com.stevesoltys.seedvault.settings.SettingsManager +import com.stevesoltys.seedvault.transport.backup.AppBackupManager import com.stevesoltys.seedvault.transport.backup.PackageService import com.stevesoltys.seedvault.worker.BackupRequester +import kotlinx.coroutines.runBlocking import org.koin.core.component.KoinComponent import org.koin.core.component.inject @@ -36,6 +38,7 @@ internal class NotificationBackupObserver( private val metadataManager: MetadataManager by inject() private val packageService: PackageService by inject() private val settingsManager: SettingsManager by inject() + private val appBackupManager: AppBackupManager by inject() private var currentPackage: String? = null private var numPackages: Int = 0 private var numPackagesToReport: Int = 0 @@ -141,6 +144,11 @@ internal class NotificationBackupObserver( Log.e(TAG, "Error getting number of all user packages: ", e) requestedPackages } + // TODO handle exceptions + runBlocking { + // TODO check if UI thread + appBackupManager.afterBackupFinished() + } nm.onBackupFinished(success, numPackagesToReport, total, size) } } diff --git a/app/src/main/java/com/stevesoltys/seedvault/worker/ApkBackup.kt b/app/src/main/java/com/stevesoltys/seedvault/worker/ApkBackup.kt index 786975c72..898cb1933 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/worker/ApkBackup.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/worker/ApkBackup.kt @@ -5,7 +5,6 @@ package com.stevesoltys.seedvault.worker -import android.annotation.SuppressLint import android.content.pm.PackageInfo import android.content.pm.PackageManager import android.content.pm.Signature @@ -13,94 +12,91 @@ import android.content.pm.SigningInfo import android.util.Log import android.util.PackageUtils.computeSha256DigestBytes import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER -import com.stevesoltys.seedvault.crypto.Crypto -import com.stevesoltys.seedvault.encodeBase64 -import com.stevesoltys.seedvault.metadata.ApkSplit -import com.stevesoltys.seedvault.metadata.MetadataManager import com.stevesoltys.seedvault.metadata.PackageMetadata +import com.stevesoltys.seedvault.proto.Snapshot import com.stevesoltys.seedvault.settings.SettingsManager +import com.stevesoltys.seedvault.transport.SnapshotManager +import com.stevesoltys.seedvault.transport.backup.AppBackupManager +import com.stevesoltys.seedvault.transport.backup.BackupReceiver +import com.stevesoltys.seedvault.transport.backup.forProto +import com.stevesoltys.seedvault.transport.backup.hexFromProto import com.stevesoltys.seedvault.transport.backup.isNotUpdatedSystemApp import com.stevesoltys.seedvault.transport.backup.isTestOnly +import org.calyxos.seedvault.core.toHexString import java.io.File import java.io.FileInputStream import java.io.IOException -import java.io.InputStream -import java.io.OutputStream -import java.security.MessageDigest private val TAG = ApkBackup::class.java.simpleName +internal const val BASE_SPLIT = "org.calyxos.seedvault.BASE_SPLIT" internal class ApkBackup( private val pm: PackageManager, - private val crypto: Crypto, + private val backupReceiver: BackupReceiver, + private val appBackupManager: AppBackupManager, + private val snapshotManager: SnapshotManager, private val settingsManager: SettingsManager, - private val metadataManager: MetadataManager, ) { + private val snapshotCreator + get() = appBackupManager.snapshotCreator ?: error("No SnapshotCreator") + /** * Checks if a new APK needs to get backed up, * because the version code or the signatures have changed. - * Only if an APK needs a backup, an [OutputStream] is obtained from the given streamGetter - * and the APK binary written to it. + * Only if APKs need backup, they get chunked and uploaded. * * @return new [PackageMetadata] if an APK backup was made or null if no backup was made. */ @Throws(IOException::class) - @SuppressLint("NewApi") // can be removed when minSdk is set to 30 - suspend fun backupApkIfNecessary( - packageInfo: PackageInfo, - streamGetter: suspend (name: String) -> OutputStream, - ): PackageMetadata? { + suspend fun backupApkIfNecessary(packageInfo: PackageInfo) { // do not back up @pm@ val packageName = packageInfo.packageName - if (packageName == MAGIC_PACKAGE_MANAGER) return null + if (packageName == MAGIC_PACKAGE_MANAGER) return // do not back up when setting is not enabled - if (!settingsManager.backupApks()) return null + if (!settingsManager.backupApks()) return // do not back up if package is blacklisted if (!settingsManager.isBackupEnabled(packageName)) { Log.d(TAG, "Package $packageName is blacklisted. Not backing it up.") - return null + return } // do not back up test-only apps as we can't re-install them anyway // see: https://commonsware.com/blog/2017/10/31/android-studio-3p0-flag-test-only.html if (packageInfo.isTestOnly()) { Log.d(TAG, "Package $packageName is test-only app. Not backing it up.") - return null + return } // do not back up system apps that haven't been updated if (packageInfo.isNotUpdatedSystemApp()) { Log.d(TAG, "Package $packageName is vanilla system app. Not backing it up.") - return null + return } // TODO remove when adding support for packages with multiple signers - val signingInfo = packageInfo.signingInfo ?: return null + val signingInfo = packageInfo.signingInfo ?: return if (signingInfo.hasMultipleSigners()) { Log.e(TAG, "Package $packageName has multiple signers. Not backing it up.") - return null + return } // get signatures - val signatures = signingInfo.getSignatures() + val signatures = signingInfo.getSignaturesHex() if (signatures.isEmpty()) { Log.e(TAG, "Package $packageName has no signatures. Not backing it up.") - return null + return } - // get cached metadata about package - val packageMetadata = metadataManager.getPackageMetadata(packageName) - ?: PackageMetadata() - - // get version codes + // get info from latest snapshot val version = packageInfo.longVersionCode - val backedUpVersion = packageMetadata.version ?: 0L // no version will cause backup + val oldApk = snapshotManager.latestSnapshot?.appsMap?.get(packageName)?.apk + val backedUpVersion = oldApk?.versionCode ?: 0L // no version will cause backup // do not backup if we have the version already and signatures did not change - if (version <= backedUpVersion && !signaturesChanged(packageMetadata, signatures)) { + if (version <= backedUpVersion && !signaturesChanged(oldApk, signatures)) { Log.d( TAG, "Package $packageName with version $version" + " already has a backup ($backedUpVersion)" + @@ -108,40 +104,52 @@ internal class ApkBackup( ) // We could also check if there are new feature module splits to back up, // but we rely on the app themselves to re-download those, if needed after restore. - return null + return } + // builder for Apk object + val apkBuilder = Snapshot.Apk.newBuilder() + .setVersionCode(version) + .setInstaller(pm.getInstallSourceInfo(packageName).installingPackageName) + .addAllSignatures(signatures.forProto()) + // get an InputStream for the APK - val sourceDir = packageInfo.applicationInfo?.sourceDir ?: return null - val inputStream = getApkInputStream(sourceDir) - // copy the APK to the storage's output and calculate SHA-256 hash while at it - val name = crypto.getNameForApk(metadataManager.salt, packageName) - val sha256 = copyStreamsAndGetHash(inputStream, streamGetter(name)) + val sourceDir = packageInfo.applicationInfo?.sourceDir ?: return + // upload the APK to the backend + getApkInputStream(sourceDir).use { inputStream -> + backupReceiver.readFromStream(inputStream) + } + val backupData = backupReceiver.finalize() + // store base split in builder + val baseSplit = Snapshot.Split.newBuilder() + .setName(BASE_SPLIT) + .addAllChunkIds(backupData.chunks.forProto()) + apkBuilder + .addSplits(baseSplit) + val chunkMap = backupData.chunkMap.toMutableMap() // back up splits if they exist - val splits = - if (packageInfo.splitNames == null) null else backupSplitApks(packageInfo, streamGetter) + val splits = if (packageInfo.splitNames == null) { + emptyList() + } else { + backupSplitApks(packageInfo, chunkMap) + } + apkBuilder.addAllSplits(splits) + val apk = apkBuilder.build() + snapshotCreator.onApkBackedUp(packageName, apk, chunkMap) Log.d(TAG, "Backed up new APK of $packageName with version ${packageInfo.versionName}.") - - // return updated metadata - return packageMetadata.copy( - version = version, - installer = pm.getInstallSourceInfo(packageName).installingPackageName, - splits = splits, - sha256 = sha256, - signatures = signatures - ) } private fun signaturesChanged( - packageMetadata: PackageMetadata, + apk: Snapshot.Apk?, signatures: List, ): Boolean { - // no signatures in package metadata counts as them not having changed - if (packageMetadata.signatures == null) return false + // no signatures counts as them not having changed + if (apk == null || apk.signaturesList.isNullOrEmpty()) return false + val sigHex = apk.signaturesList.hexFromProto() // TODO to support multiple signers check if lists differ - return packageMetadata.signatures.intersect(signatures).isEmpty() + return sigHex.intersect(signatures.toSet()).isEmpty() } @Throws(IOException::class) @@ -159,8 +167,8 @@ internal class ApkBackup( @Throws(IOException::class) private suspend fun backupSplitApks( packageInfo: PackageInfo, - streamGetter: suspend (name: String) -> OutputStream, - ): List { + chunkMap: MutableMap, + ): List { check(packageInfo.splitNames != null) // attention: though not documented, splitSourceDirs can be null val splitSourceDirs = packageInfo.applicationInfo?.splitSourceDirs ?: emptyArray() @@ -169,97 +177,42 @@ internal class ApkBackup( "splitNames is ${packageInfo.splitNames.toList()}, " + "but splitSourceDirs is ${splitSourceDirs.toList()}" } - val splits = ArrayList(packageInfo.splitNames.size) + val splits = ArrayList(packageInfo.splitNames.size) for (i in packageInfo.splitNames.indices) { - val split = backupSplitApk( - packageName = packageInfo.packageName, - splitName = packageInfo.splitNames[i], - sourceDir = splitSourceDirs[i], - streamGetter = streamGetter - ) + // copy the split APK to the storage stream + getApkInputStream(splitSourceDirs[i]).use { inputStream -> + backupReceiver.readFromStream(inputStream) + } + val backupData = backupReceiver.finalize() + val split = Snapshot.Split.newBuilder() + .setName(packageInfo.splitNames[i]) + .addAllChunkIds(backupData.chunks.forProto()) + .build() splits.add(split) + chunkMap.putAll(backupData.chunkMap) } return splits } - @Throws(IOException::class) - private suspend fun backupSplitApk( - packageName: String, - splitName: String, - sourceDir: String, - streamGetter: suspend (name: String) -> OutputStream, - ): ApkSplit { - // Calculate sha256 hash first to determine file name suffix. - // We could also just use the split name as a suffix, but there is a theoretical risk - // that we exceed the maximum file name length, so we use the hash instead. - // The downside is that we need to read the file two times. - val messageDigest = MessageDigest.getInstance("SHA-256") - val size = getApkInputStream(sourceDir).use { inputStream -> - val buffer = ByteArray(DEFAULT_BUFFER_SIZE) - var readCount = 0 - var bytes = inputStream.read(buffer) - while (bytes >= 0) { - readCount += bytes - messageDigest.update(buffer, 0, bytes) - bytes = inputStream.read(buffer) - } - readCount - } - val sha256 = messageDigest.digest().encodeBase64() - val name = crypto.getNameForApk(metadataManager.salt, packageName, splitName) - // copy the split APK to the storage stream - getApkInputStream(sourceDir).use { inputStream -> - streamGetter(name).use { outputStream -> - inputStream.copyTo(outputStream) - } - } - return ApkSplit(splitName, size.toLong(), sha256) - } - -} - -/** - * Copy the APK from the given [InputStream] to the given [OutputStream] - * and calculate the SHA-256 hash while at it. - * - * Both streams will be closed when the method returns. - * - * @return the APK's SHA-256 hash in Base64 format. - */ -@Throws(IOException::class) -fun copyStreamsAndGetHash(inputStream: InputStream, outputStream: OutputStream): String { - val messageDigest = MessageDigest.getInstance("SHA-256") - outputStream.use { oStream -> - inputStream.use { inputStream -> - val buffer = ByteArray(DEFAULT_BUFFER_SIZE) - var bytes = inputStream.read(buffer) - while (bytes >= 0) { - oStream.write(buffer, 0, bytes) - messageDigest.update(buffer, 0, bytes) - bytes = inputStream.read(buffer) - } - } - } - return messageDigest.digest().encodeBase64() } /** - * Returns a list of Base64 encoded SHA-256 signature hashes. + * Returns a list of lowercase hex encoded SHA-256 signature hashes. */ -fun SigningInfo?.getSignatures(): List { +fun SigningInfo?.getSignaturesHex(): List { return if (this == null) { emptyList() } else if (hasMultipleSigners()) { apkContentsSigners.map { signature -> - hashSignature(signature).encodeBase64() + hashSignature(signature).toHexString() } } else { signingCertificateHistory.map { signature -> - hashSignature(signature).encodeBase64() + hashSignature(signature).toHexString() } } } -private fun hashSignature(signature: Signature): ByteArray { +internal fun hashSignature(signature: Signature): ByteArray { return computeSha256DigestBytes(signature.toByteArray()) ?: throw AssertionError() } diff --git a/app/src/main/java/com/stevesoltys/seedvault/worker/ApkBackupManager.kt b/app/src/main/java/com/stevesoltys/seedvault/worker/ApkBackupManager.kt index 3311a728f..5d7723337 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/worker/ApkBackupManager.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/worker/ApkBackupManager.kt @@ -8,19 +8,15 @@ package com.stevesoltys.seedvault.worker import android.content.Context import android.content.pm.PackageInfo import android.util.Log +import com.stevesoltys.seedvault.backend.isOutOfSpace import com.stevesoltys.seedvault.metadata.MetadataManager import com.stevesoltys.seedvault.metadata.PackageState.NOT_ALLOWED import com.stevesoltys.seedvault.metadata.PackageState.WAS_STOPPED -import com.stevesoltys.seedvault.backend.BackendManager -import com.stevesoltys.seedvault.backend.getMetadataOutputStream -import com.stevesoltys.seedvault.backend.isOutOfSpace import com.stevesoltys.seedvault.settings.SettingsManager import com.stevesoltys.seedvault.transport.backup.PackageService import com.stevesoltys.seedvault.transport.backup.isStopped import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager import com.stevesoltys.seedvault.ui.notification.getAppName -import kotlinx.coroutines.delay -import org.calyxos.seedvault.core.backends.LegacyAppBackupFile import java.io.IOException internal class ApkBackupManager( @@ -30,7 +26,6 @@ internal class ApkBackupManager( private val packageService: PackageService, private val iconManager: IconManager, private val apkBackup: ApkBackup, - private val backendManager: BackendManager, private val nm: BackupNotificationManager, ) { @@ -51,14 +46,6 @@ internal class ApkBackupManager( backUpApks() } } finally { - keepTrying { - // upload all local changes only at the end, - // so we don't have to re-upload the metadata - val token = settingsManager.getToken() ?: error("no token") - backendManager.backend.getMetadataOutputStream(token).use { outputStream -> - metadataManager.uploadMetadata(outputStream) - } - } nm.onApkBackupDone() } } @@ -100,47 +87,22 @@ internal class ApkBackupManager( private suspend fun uploadIcons() { try { - val token = settingsManager.getToken() ?: throw IOException("no current token") - val handle = LegacyAppBackupFile.IconsFile(token) - backendManager.backend.save(handle).use { - iconManager.uploadIcons(token, it) - } - } catch (e: IOException) { + iconManager.uploadIcons() + } catch (e: Exception) { Log.e(TAG, "Error uploading icons: ", e) } } /** - * Backs up an APK for the given [PackageInfo]. - * - * @return true if a backup was performed and false if no backup was needed or it failed. + * Backs up one (or more split) APK(s) for the given [PackageInfo], if needed. */ - private suspend fun backUpApk(packageInfo: PackageInfo): Boolean { + private suspend fun backUpApk(packageInfo: PackageInfo) { val packageName = packageInfo.packageName - return try { - apkBackup.backupApkIfNecessary(packageInfo) { name -> - val token = settingsManager.getToken() ?: throw IOException("no current token") - backendManager.backend.save(LegacyAppBackupFile.Blob(token, name)) - }?.let { packageMetadata -> - metadataManager.onApkBackedUp(packageInfo, packageMetadata) - true - } ?: false + try { + apkBackup.backupApkIfNecessary(packageInfo) } catch (e: IOException) { Log.e(TAG, "Error while writing APK for $packageName", e) if (e.isOutOfSpace()) nm.onInsufficientSpaceError() - false - } - } - - private suspend fun keepTrying(n: Int = 3, block: suspend () -> Unit) { - for (i in 1..n) { - try { - block() - } catch (e: Exception) { - if (i == n) throw e - Log.e(TAG, "Error (#$i), we'll keep trying", e) - delay(1000) - } } } } diff --git a/app/src/main/java/com/stevesoltys/seedvault/worker/AppBackupWorker.kt b/app/src/main/java/com/stevesoltys/seedvault/worker/AppBackupWorker.kt index b7041901e..e97ff922d 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/worker/AppBackupWorker.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/worker/AppBackupWorker.kt @@ -24,6 +24,7 @@ import androidx.work.WorkManager import androidx.work.WorkerParameters import com.stevesoltys.seedvault.backend.BackendManager import com.stevesoltys.seedvault.settings.SettingsManager +import com.stevesoltys.seedvault.transport.backup.AppBackupManager import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager import com.stevesoltys.seedvault.ui.notification.NOTIFICATION_ID_OBSERVER import org.koin.core.component.KoinComponent @@ -101,6 +102,7 @@ class AppBackupWorker( private val backupRequester: BackupRequester by inject() private val settingsManager: SettingsManager by inject() private val apkBackupManager: ApkBackupManager by inject() + private val appBackupManager: AppBackupManager by inject() private val backendManager: BackendManager by inject() private val nm: BackupNotificationManager by inject() @@ -137,6 +139,15 @@ class AppBackupWorker( private suspend fun doBackup(): Result { var result: Result = Result.success() + if (!isStopped) { + Log.i(TAG, "Initializing backup info...") + try { + appBackupManager.beforeBackup() + } catch (e: Exception) { + Log.e(TAG, "Error populating blobs cache: ", e) + return Result.retry() + } + } try { Log.i(TAG, "Starting APK backup... (stopped: $isStopped)") if (!isStopped) apkBackupManager.backup() diff --git a/app/src/main/java/com/stevesoltys/seedvault/worker/IconManager.kt b/app/src/main/java/com/stevesoltys/seedvault/worker/IconManager.kt index 821a527d8..2a1906b6e 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/worker/IconManager.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/worker/IconManager.kt @@ -6,7 +6,7 @@ package com.stevesoltys.seedvault.worker import android.content.Context -import android.graphics.Bitmap.CompressFormat.WEBP_LOSSY +import android.graphics.Bitmap.CompressFormat.JPEG import android.graphics.BitmapFactory import android.graphics.drawable.Drawable import android.util.Log @@ -15,25 +15,26 @@ import androidx.appcompat.content.res.AppCompatResources.getDrawable import androidx.core.graphics.drawable.toBitmap import androidx.core.graphics.drawable.toDrawable import com.stevesoltys.seedvault.R -import com.stevesoltys.seedvault.crypto.Crypto -import com.stevesoltys.seedvault.crypto.TYPE_ICONS -import com.stevesoltys.seedvault.header.VERSION +import com.stevesoltys.seedvault.proto.Snapshot +import com.stevesoltys.seedvault.transport.backup.AppBackupManager +import com.stevesoltys.seedvault.transport.backup.BackupReceiver import com.stevesoltys.seedvault.transport.backup.PackageService +import com.stevesoltys.seedvault.transport.restore.Loader import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext -import org.calyxos.backup.storage.crypto.StreamCrypto.toByteArray +import org.calyxos.seedvault.core.backends.AppBackupFileType +import org.calyxos.seedvault.core.toHexString +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream import java.io.File import java.io.IOException -import java.io.InputStream -import java.io.OutputStream -import java.nio.ByteBuffer +import java.nio.file.attribute.FileTime import java.security.GeneralSecurityException -import java.util.zip.Deflater.BEST_SPEED +import java.util.zip.Deflater.NO_COMPRESSION import java.util.zip.ZipEntry import java.util.zip.ZipInputStream import java.util.zip.ZipOutputStream -internal const val FILE_BACKUP_ICONS = ".backup.icons" private const val ICON_SIZE = 128 private const val ICON_QUALITY = 75 private const val CACHE_FOLDER = "restore-icons" @@ -42,63 +43,87 @@ private val TAG = IconManager::class.simpleName internal class IconManager( private val context: Context, private val packageService: PackageService, - private val crypto: Crypto, + private val backupReceiver: BackupReceiver, + private val loader: Loader, + private val appBackupManager: AppBackupManager, ) { + private val snapshotCreator + get() = appBackupManager.snapshotCreator ?: error("No SnapshotCreator") + @Throws(IOException::class, GeneralSecurityException::class) - fun uploadIcons(token: Long, outputStream: OutputStream) { + suspend fun uploadIcons() { Log.d(TAG, "Start uploading icons") val packageManager = context.packageManager - crypto.newEncryptingStreamV1(outputStream, getAD(VERSION, token)).use { cryptoStream -> - ZipOutputStream(cryptoStream).use { zip -> - zip.setLevel(BEST_SPEED) - val entries = mutableSetOf() - packageService.allUserPackages.forEach { - val applicationInfo = it.applicationInfo ?: return@forEach - val drawable = packageManager.getApplicationIcon(applicationInfo) - if (packageManager.isDefaultApplicationIcon(drawable)) return@forEach - val entry = ZipEntry(it.packageName) - zip.putNextEntry(entry) - drawable.toBitmap(ICON_SIZE, ICON_SIZE).compress(WEBP_LOSSY, ICON_QUALITY, zip) - entries.add(it.packageName) - zip.closeEntry() + val byteArrayOutputStream = ByteArrayOutputStream() + ZipOutputStream(byteArrayOutputStream).use { zip -> + zip.setLevel(NO_COMPRESSION) // we compress with zstd after chunking the zip + val entries = mutableSetOf() + // sort packages by package name to get deterministic ZIP + packageService.allUserPackages.sortedBy { it.packageName }.forEach { + val applicationInfo = it.applicationInfo ?: return@forEach + val drawable = packageManager.getApplicationIcon(applicationInfo) + if (packageManager.isDefaultApplicationIcon(drawable)) return@forEach + val entry = ZipEntry(it.packageName).apply { + // needed to be deterministic + setLastModifiedTime(FileTime.fromMillis(0)) } - packageService.launchableSystemApps.forEach { - val drawable = it.loadIcon(packageManager) - if (packageManager.isDefaultApplicationIcon(drawable)) return@forEach - // check for duplicates (e.g. updated launchable system app) - if (it.activityInfo.packageName in entries) return@forEach - val entry = ZipEntry(it.activityInfo.packageName) - zip.putNextEntry(entry) - drawable.toBitmap(ICON_SIZE, ICON_SIZE).compress(WEBP_LOSSY, ICON_QUALITY, zip) - zip.closeEntry() + zip.putNextEntry(entry) + // WEBP_LOSSY compression wasn't deterministic in our tests, so use JPEG + drawable.toBitmap(ICON_SIZE, ICON_SIZE).compress(JPEG, ICON_QUALITY, zip) + entries.add(it.packageName) + zip.closeEntry() + } + // sort packages by package name to get deterministic ZIP + packageService.launchableSystemApps.sortedBy { it.activityInfo.packageName }.forEach { + val drawable = it.loadIcon(packageManager) + if (packageManager.isDefaultApplicationIcon(drawable)) return@forEach + // check for duplicates (e.g. updated launchable system app) + if (it.activityInfo.packageName in entries) return@forEach + val entry = ZipEntry(it.activityInfo.packageName).apply { + // needed to be deterministic + setLastModifiedTime(FileTime.fromMillis(0)) } + zip.putNextEntry(entry) + // WEBP_LOSSY compression wasn't deterministic in our tests, so use JPEG + drawable.toBitmap(ICON_SIZE, ICON_SIZE).compress(JPEG, ICON_QUALITY, zip) + zip.closeEntry() } } + backupReceiver.addBytes(byteArrayOutputStream.toByteArray()) + val backupData = backupReceiver.finalize() + snapshotCreator.onIconsBackedUp(backupData) Log.d(TAG, "Finished uploading icons") } /** - * Downloads icons file from given [inputStream]. + * Downloads icons file from given [snapshot] from the repository with [repoId]. * @return a set of package names for which icons were found */ @Throws(IOException::class, SecurityException::class, GeneralSecurityException::class) - fun downloadIcons(version: Byte, token: Long, inputStream: InputStream): Set { + suspend fun downloadIcons(repoId: String, snapshot: Snapshot): Set { Log.d(TAG, "Start downloading icons") val folder = File(context.cacheDir, CACHE_FOLDER) if (!folder.isDirectory && !folder.mkdirs()) throw IOException("Can't create cache folder for icons") + + val outputStream = ByteArrayOutputStream() + snapshot.iconChunkIdsList.forEach { + val blob = snapshot.getBlobsOrThrow(it.toByteArray().toHexString()) + val handle = AppBackupFileType.Blob(repoId, blob.id.toByteArray().toHexString()) + loader.loadFile(handle).use { inputStream -> + inputStream.copyTo(outputStream) + } + } val set = mutableSetOf() - crypto.newDecryptingStreamV1(inputStream, getAD(version, token)).use { cryptoStream -> - ZipInputStream(cryptoStream).use { zip -> - var entry = zip.nextEntry - while (entry != null) { - File(folder, entry.name).outputStream().use { outputStream -> - zip.copyTo(outputStream) - } - set.add(entry.name) - entry = zip.nextEntry + ZipInputStream(ByteArrayInputStream(outputStream.toByteArray())).use { zip -> + var entry = zip.nextEntry + while (entry != null) { + File(folder, entry.name).outputStream().use { outputStream -> + zip.copyTo(outputStream) } + set.add(entry.name) + entry = zip.nextEntry } } Log.d(TAG, "Finished downloading icons") @@ -142,10 +167,4 @@ internal class IconManager( Log.e(TAG, "Could delete icons: $result") } - private fun getAD(version: Byte, token: Long) = ByteBuffer.allocate(2 + 8) - .put(version) - .put(TYPE_ICONS) - .put(token.toByteArray()) - .array() - } diff --git a/app/src/main/java/com/stevesoltys/seedvault/worker/WorkerModule.kt b/app/src/main/java/com/stevesoltys/seedvault/worker/WorkerModule.kt index dea0b9410..65e1173b1 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/worker/WorkerModule.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/worker/WorkerModule.kt @@ -5,6 +5,7 @@ package com.stevesoltys.seedvault.worker +import com.stevesoltys.seedvault.transport.backup.AppBackupManager import org.koin.android.ext.koin.androidContext import org.koin.dsl.module @@ -20,15 +21,19 @@ val workerModule = module { IconManager( context = androidContext(), packageService = get(), - crypto = get(), + backupReceiver = get(), + loader = get(), + appBackupManager = get(), ) } + single { AppBackupManager(get(), get(), get()) } single { ApkBackup( pm = androidContext().packageManager, - crypto = get(), + backupReceiver = get(), + appBackupManager = get(), + snapshotManager = get(), settingsManager = get(), - metadataManager = get() ) } single { @@ -39,7 +44,6 @@ val workerModule = module { packageService = get(), apkBackup = get(), iconManager = get(), - backendManager = get(), nm = get() ) } diff --git a/app/src/main/proto/snapshot.proto b/app/src/main/proto/snapshot.proto index 80792d863..4885ca45d 100644 --- a/app/src/main/proto/snapshot.proto +++ b/app/src/main/proto/snapshot.proto @@ -32,6 +32,9 @@ message Snapshot { } message Apk { + /** + * Attention: Has default value of 0 + */ uint64 versionCode = 1; string installer = 2; repeated bytes signatures = 3; diff --git a/app/src/test/java/com/stevesoltys/seedvault/restore/AppSelectionManagerTest.kt b/app/src/test/java/com/stevesoltys/seedvault/restore/AppSelectionManagerTest.kt index 79d5945cc..bd76c6ff5 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/restore/AppSelectionManagerTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/restore/AppSelectionManagerTest.kt @@ -9,11 +9,11 @@ import androidx.test.ext.junit.runners.AndroidJUnit4 import app.cash.turbine.TurbineTestContext import app.cash.turbine.test import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER +import com.stevesoltys.seedvault.backend.BackendManager import com.stevesoltys.seedvault.getRandomString import com.stevesoltys.seedvault.metadata.BackupMetadata import com.stevesoltys.seedvault.metadata.PackageMetadata import com.stevesoltys.seedvault.metadata.PackageMetadataMap -import com.stevesoltys.seedvault.backend.BackendManager import com.stevesoltys.seedvault.transport.TransportTest import com.stevesoltys.seedvault.ui.PACKAGE_NAME_CONTACTS import com.stevesoltys.seedvault.ui.PACKAGE_NAME_SETTINGS @@ -35,7 +35,6 @@ import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Assertions.fail import org.junit.runner.RunWith -import java.io.ByteArrayInputStream import java.io.IOException import kotlin.random.Random @@ -66,7 +65,8 @@ internal class AppSelectionManagerTest : TransportTest() { ) @Test - fun `apps without backup and APK, as well as system apps are filtered out`() = runTest { + fun `apps without backup and APK, as well as system apps are filtered out`() = scope.runTest { + expectIconLoading(emptySet()) appSelectionManager.selectedAppsFlow.test { val initialState = awaitItem() assertEquals(emptyList(), initialState.apps) @@ -91,11 +91,15 @@ internal class AppSelectionManagerTest : TransportTest() { assertEquals(PACKAGE_NAME_SYSTEM, initialApps.apps[0].packageName) assertTrue(initialApps.allSelected) assertFalse(initialApps.iconsLoaded) + + // now icons have loaded and apps were updated + awaitItem() } } @Test - fun `apps get sorted by name, special items on top`() = runTest { + fun `apps get sorted by name, special items on top`() = scope.runTest { + expectIconLoading(emptySet()) appSelectionManager.selectedAppsFlow.test { awaitItem() @@ -124,11 +128,15 @@ internal class AppSelectionManagerTest : TransportTest() { assertEquals(PACKAGE_NAME_SYSTEM, initialApps.apps[1].packageName) assertEquals(packageName2, initialApps.apps[2].packageName) assertEquals(packageName1, initialApps.apps[3].packageName) + + // now icons have loaded and apps were updated + awaitItem() } } @Test - fun `test app selection`() = runTest { + fun `test app selection`() = scope.runTest { + expectIconLoading() appSelectionManager.selectedAppsFlow.test { awaitItem() @@ -146,6 +154,9 @@ internal class AppSelectionManagerTest : TransportTest() { initialApps.apps.forEach { assertTrue(it.selected) } assertTrue(initialApps.allSelected) + // now icons have loaded and apps were updated + awaitItem() + // deselect last app in list appSelectionManager.onAppSelected(initialApps.apps[2]) val oneDeselected = awaitItem() @@ -248,7 +259,7 @@ internal class AppSelectionManagerTest : TransportTest() { } @Test - fun `finishing selection filters unselected apps, leaves system apps`() = runTest { + fun `finishing selection filters unselected apps, leaves system apps`() = scope.runTest { testFiltering { backup -> val itemsWithIcons = awaitItem() @@ -283,48 +294,50 @@ internal class AppSelectionManagerTest : TransportTest() { } @Test - fun `finishing selection without system apps only removes non-special system apps`() = runTest { - testFiltering { backup -> - val itemsWithIcons = awaitItem() - - // unselect all system apps and settings, contacts should stay - val systemMeta = itemsWithIcons.apps.find { it.packageName == PACKAGE_NAME_SYSTEM } - ?: fail() - val settings = itemsWithIcons.apps.find { it.packageName == PACKAGE_NAME_SETTINGS } - ?: fail() - appSelectionManager.onAppSelected(systemMeta) - awaitItem() - appSelectionManager.onAppSelected(settings) - - // assert that both apps are unselected - val finalSelection = awaitItem() - // we have 6 real apps (two are hidden) plus system meta item, makes 5 - assertEquals(5, finalSelection.apps.size) - finalSelection.apps.forEach { - if (it.packageName in listOf(PACKAGE_NAME_SYSTEM, PACKAGE_NAME_SETTINGS)) { - assertFalse(it.selected) - } else { - assertTrue(it.selected) + fun `finishing selection without system apps only removes non-special system apps`() = + scope.runTest { + testFiltering { backup -> + val itemsWithIcons = awaitItem() + + // unselect all system apps and settings, contacts should stay + val systemMeta = itemsWithIcons.apps.find { it.packageName == PACKAGE_NAME_SYSTEM } + ?: fail() + val settings = itemsWithIcons.apps.find { it.packageName == PACKAGE_NAME_SETTINGS } + ?: fail() + appSelectionManager.onAppSelected(systemMeta) + awaitItem() + appSelectionManager.onAppSelected(settings) + + // assert that both apps are unselected + val finalSelection = awaitItem() + // we have 6 real apps (two are hidden) plus system meta item, makes 5 + assertEquals(5, finalSelection.apps.size) + finalSelection.apps.forEach { + if (it.packageName in listOf(PACKAGE_NAME_SYSTEM, PACKAGE_NAME_SETTINGS)) { + assertFalse(it.selected) + } else { + assertTrue(it.selected) + } } - } - // 4 apps should survive: app1, app2, app4 (hidden) and contacts - val filteredBackup = appSelectionManager.onAppSelectionFinished(backup) - assertEquals(4, filteredBackup.packageMetadataMap.size) - assertEquals( - setOf(packageName1, packageName2, packageName4, PACKAGE_NAME_CONTACTS), - filteredBackup.packageMetadataMap.keys, - ) + // 4 apps should survive: app1, app2, app4 (hidden) and contacts + val filteredBackup = appSelectionManager.onAppSelectionFinished(backup) + assertEquals(4, filteredBackup.packageMetadataMap.size) + assertEquals( + setOf(packageName1, packageName2, packageName4, PACKAGE_NAME_CONTACTS), + filteredBackup.packageMetadataMap.keys, + ) + } } - } @Test - fun `system apps only pre-selected in setup wizard`() = runTest { + fun `system apps only pre-selected in setup wizard`() = scope.runTest { val backup = getRestorableBackup( mutableMapOf( packageName1 to PackageMetadata(system = true, isLaunchableSystemApp = false), ) ) + expectIconLoading(emptySet()) // choose restore set in setup wizard appSelectionManager.selectedAppsFlow.test { awaitItem() @@ -334,6 +347,9 @@ internal class AppSelectionManagerTest : TransportTest() { assertEquals(1, initialApps.apps.size) assertEquals(PACKAGE_NAME_SYSTEM, initialApps.apps[0].packageName) assertTrue(initialApps.apps[0].selected) // system settings is selected + + // now icons have loaded and apps were updated + awaitItem() } appSelectionManager.selectedAppsFlow.test { awaitItem() @@ -343,11 +359,15 @@ internal class AppSelectionManagerTest : TransportTest() { assertEquals(1, initialApps.apps.size) assertEquals(PACKAGE_NAME_SYSTEM, initialApps.apps[0].packageName) assertFalse(initialApps.apps[0].selected) // system settings is NOT selected + + // now icons have loaded and apps were updated + awaitItem() } } @Test - fun `@pm@ doesn't get filtered out`() = runTest { + fun `@pm@ doesn't get filtered out`() = scope.runTest { + expectIconLoading(emptySet()) appSelectionManager.selectedAppsFlow.test { awaitItem() @@ -366,6 +386,9 @@ internal class AppSelectionManagerTest : TransportTest() { assertEquals(1, initialApps.apps.size) assertEquals(PACKAGE_NAME_SYSTEM, initialApps.apps[0].packageName) + // now icons have loaded and apps were updated + awaitItem() + // actual filtered backup includes @pm@ only val filteredBackup = appSelectionManager.onAppSelectionFinished(backup) assertEquals(1, filteredBackup.packageMetadataMap.size) @@ -423,15 +446,8 @@ internal class AppSelectionManagerTest : TransportTest() { } private fun expectIconLoading(icons: Set = setOf(packageName1, packageName2)) { - val backend: Backend = mockk() - val inputStream = ByteArrayInputStream(Random.nextBytes(42)) - every { backendManager.backend } returns backend - coEvery { - backend.load(LegacyAppBackupFile.IconsFile(backupMetadata.token)) - } returns inputStream - every { - iconManager.downloadIcons(backupMetadata.version, backupMetadata.token, inputStream) - } returns icons + // TODO adapt to new code + coEvery { iconManager.downloadIcons(any(), any()) } returns icons } } diff --git a/app/src/test/java/com/stevesoltys/seedvault/restore/install/ApkBackupRestoreTest.kt b/app/src/test/java/com/stevesoltys/seedvault/restore/install/ApkBackupRestoreTest.kt index d98241205..7680f049e 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/restore/install/ApkBackupRestoreTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/restore/install/ApkBackupRestoreTest.kt @@ -13,29 +13,43 @@ import android.content.pm.Signature import android.graphics.drawable.Drawable import android.util.PackageUtils import app.cash.turbine.test +import com.google.protobuf.ByteString +import com.google.protobuf.ByteString.copyFromUtf8 import com.stevesoltys.seedvault.BackupStateManager import com.stevesoltys.seedvault.assertReadEquals +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.backend.LegacyStoragePlugin +import com.stevesoltys.seedvault.decodeBase64 import com.stevesoltys.seedvault.getRandomString -import com.stevesoltys.seedvault.metadata.ApkSplit import com.stevesoltys.seedvault.metadata.PackageMetadata import com.stevesoltys.seedvault.metadata.PackageMetadataMap -import com.stevesoltys.seedvault.backend.LegacyStoragePlugin -import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.proto.Snapshot import com.stevesoltys.seedvault.restore.RestorableBackup import com.stevesoltys.seedvault.restore.install.ApkInstallState.IN_PROGRESS import com.stevesoltys.seedvault.restore.install.ApkInstallState.QUEUED import com.stevesoltys.seedvault.restore.install.ApkInstallState.SUCCEEDED +import com.stevesoltys.seedvault.transport.SnapshotManager import com.stevesoltys.seedvault.transport.TransportTest +import com.stevesoltys.seedvault.transport.backup.AppBackupManager +import com.stevesoltys.seedvault.transport.backup.BackupData +import com.stevesoltys.seedvault.transport.backup.BackupReceiver +import com.stevesoltys.seedvault.transport.backup.SnapshotCreator +import com.stevesoltys.seedvault.transport.backup.hexFromProto +import com.stevesoltys.seedvault.transport.restore.Loader import com.stevesoltys.seedvault.worker.ApkBackup +import com.stevesoltys.seedvault.worker.BASE_SPLIT +import io.mockk.Runs import io.mockk.coEvery import io.mockk.every +import io.mockk.just import io.mockk.mockk import io.mockk.mockkStatic import io.mockk.slot import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.runBlocking +import org.calyxos.seedvault.core.backends.AppBackupFileType import org.calyxos.seedvault.core.backends.Backend -import org.calyxos.seedvault.core.backends.LegacyAppBackupFile +import org.calyxos.seedvault.core.toHexString import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse @@ -47,7 +61,7 @@ import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.File import java.io.FileInputStream -import java.io.OutputStream +import java.io.InputStream import java.nio.file.Path import kotlin.random.Random @@ -62,6 +76,11 @@ internal class ApkBackupRestoreTest : TransportTest() { private val backendManager: BackendManager = mockk() private val backupManager: IBackupManager = mockk() private val backupStateManager: BackupStateManager = mockk() + private val backupReceiver: BackupReceiver = mockk() + private val appBackupManager: AppBackupManager = mockk() + private val snapshotManager: SnapshotManager = mockk() + private val snapshotCreator: SnapshotCreator = mockk() + private val loader: Loader = mockk() @Suppress("Deprecation") private val legacyStoragePlugin: LegacyStoragePlugin = mockk() @@ -70,12 +89,14 @@ internal class ApkBackupRestoreTest : TransportTest() { private val apkInstaller: ApkInstaller = mockk() private val installRestriction: InstallRestriction = mockk() - private val apkBackup = ApkBackup(pm, crypto, settingsManager, metadataManager) + private val apkBackup = + ApkBackup(pm, backupReceiver, appBackupManager, snapshotManager, settingsManager) private val apkRestore: ApkRestore = ApkRestore( context = strictContext, backupManager = backupManager, backupStateManager = backupStateManager, backendManager = backendManager, + loader = loader, legacyStoragePlugin = legacyStoragePlugin, crypto = crypto, splitCompatChecker = splitCompatChecker, @@ -89,29 +110,46 @@ internal class ApkBackupRestoreTest : TransportTest() { private val packageName: String = packageInfo.packageName private val splitName = getRandomString() private val splitBytes = byteArrayOf(0x07, 0x08, 0x09) - private val splitSha256 = "ZqZ1cVH47lXbEncWx-Pc4L6AdLZOIO2lQuXB5GypxB4" - private val packageMetadata = PackageMetadata( - time = Random.nextLong(), - version = packageInfo.longVersionCode - 1, - installer = getRandomString(), - sha256 = "eHx5jjmlvBkQNVuubQzYejay4Q_QICqD47trAF2oNHI", - signatures = listOf("AwIB"), - splits = listOf(ApkSplit(splitName, Random.nextLong(), splitSha256)) - ) - private val packageMetadataMap: PackageMetadataMap = hashMapOf(packageName to packageMetadata) - private val installerName = packageMetadata.installer + private val apkChunkId = Random.nextBytes(32).toHexString() + private val splitChunkId = Random.nextBytes(32).toHexString() + private val apkBlob = + Snapshot.Blob.newBuilder().setId(ByteString.copyFrom(Random.nextBytes(32))).build() + private val splitBlob = + Snapshot.Blob.newBuilder().setId(ByteString.copyFrom(Random.nextBytes(32))).build() + private val apkBackupData = BackupData(listOf(apkChunkId), mapOf(apkChunkId to apkBlob)) + private val splitBackupData = BackupData(listOf(splitChunkId), mapOf(splitChunkId to splitBlob)) + private val chunkMap = apkBackupData.chunkMap + splitBackupData.chunkMap + private val baseSplit = Snapshot.Split.newBuilder().setName(BASE_SPLIT) + .addAllChunkIds(listOf(ByteString.fromHex(apkChunkId))).build() + private val apkSplit = Snapshot.Split.newBuilder().setName(splitName) + .addAllChunkIds(listOf(ByteString.fromHex(splitChunkId))).build() + private val apk = Snapshot.Apk.newBuilder() + .setVersionCode(packageInfo.longVersionCode - 1) + .setInstaller(getRandomString()) + .addAllSignatures(mutableListOf(copyFromUtf8("AwIB".decodeBase64()))) + .addSplits(baseSplit) + .addSplits(apkSplit) + .build() + private val app = Snapshot.App.newBuilder() + .setApk(apk) + .build() + private val snapshot = Snapshot.newBuilder() + .setToken(token) + .putApps(packageName, app) + .putAllBlobs(chunkMap) + .build() + private val packageMetadataMap: PackageMetadataMap = + hashMapOf(packageName to PackageMetadata.fromSnapshot(app)) + private val installerName = apk.installer private val icon: Drawable = mockk() private val appName = getRandomString() - private val suffixName = getRandomString() private val outputStream = ByteArrayOutputStream() private val splitOutputStream = ByteArrayOutputStream() - private val outputStreamGetter: suspend (name: String) -> OutputStream = { name -> - if (name == this.name) outputStream else splitOutputStream - } init { mockkStatic(PackageUtils::class) every { backendManager.backend } returns backend + every { appBackupManager.snapshotCreator } returns snapshotCreator } @Test @@ -127,22 +165,26 @@ internal class ApkBackupRestoreTest : TransportTest() { assertTrue(createNewFile()) writeBytes(splitBytes) }.absolutePath) + val capturedApkStream = slot() every { settingsManager.isBackupEnabled(any()) } returns true every { settingsManager.backupApks() } returns true every { sigInfo.hasMultipleSigners() } returns false every { sigInfo.signingCertificateHistory } returns sigs every { PackageUtils.computeSha256DigestBytes(signatureBytes) } returns signatureHash - every { - metadataManager.getPackageMetadata(packageInfo.packageName) - } returns packageMetadata + every { snapshotManager.latestSnapshot } returns snapshot every { pm.getInstallSourceInfo(packageInfo.packageName) } returns mockk(relaxed = true) - every { metadataManager.salt } returns salt - every { crypto.getNameForApk(salt, packageName) } returns name - every { crypto.getNameForApk(salt, packageName, splitName) } returns suffixName - every { backend.providerPackageName } returns storageProviderPackageName + coEvery { backupReceiver.readFromStream(capture(capturedApkStream)) } answers { + capturedApkStream.captured.copyTo(outputStream) + } andThenAnswer { + capturedApkStream.captured.copyTo(splitOutputStream) + } + coEvery { backupReceiver.finalize() } returns apkBackupData andThen splitBackupData + every { + snapshotCreator.onApkBackedUp(packageName, any(), chunkMap) + } just Runs - apkBackup.backupApkIfNecessary(packageInfo, outputStreamGetter) + apkBackup.backupApkIfNecessary(packageInfo) assertArrayEquals(apkBytes, outputStream.toByteArray()) assertArrayEquals(splitBytes, splitOutputStream.toByteArray()) @@ -151,23 +193,23 @@ internal class ApkBackupRestoreTest : TransportTest() { val splitInputStream = ByteArrayInputStream(splitBytes) val apkPath = slot() val cacheFiles = slot>() + val repoId = getRandomString() + val apkHandle = AppBackupFileType.Blob(repoId, apkBlob.id.hexFromProto()) + val splitHandle = AppBackupFileType.Blob(repoId, splitBlob.id.hexFromProto()) + every { backend.providerPackageName } returns storageProviderPackageName every { installRestriction.isAllowedToInstallApks() } returns true every { backupStateManager.isAutoRestoreEnabled } returns false every { pm.getPackageInfo(packageName, any()) } throws NameNotFoundException() every { strictContext.cacheDir } returns tmpFile - every { crypto.getNameForApk(salt, packageName, "") } returns name - coEvery { backend.load(LegacyAppBackupFile.Blob(token, name)) } returns inputStream + coEvery { loader.loadFiles(listOf(apkHandle)) } returns inputStream every { pm.getPackageArchiveInfo(capture(apkPath), any()) } returns packageInfo every { applicationInfo.loadIcon(pm) } returns icon every { pm.getApplicationLabel(packageInfo.applicationInfo) } returns appName every { splitCompatChecker.isCompatible(metadata.deviceName, listOf(splitName)) } returns true - every { crypto.getNameForApk(salt, packageName, splitName) } returns suffixName - coEvery { - backend.load(LegacyAppBackupFile.Blob(token, suffixName)) - } returns splitInputStream + coEvery { loader.loadFiles(listOf(splitHandle)) } returns splitInputStream val resultMap = mapOf( packageName to ApkInstallResult( packageName, @@ -179,7 +221,11 @@ internal class ApkBackupRestoreTest : TransportTest() { apkInstaller.install(capture(cacheFiles), packageName, installerName, any()) } returns InstallResult(resultMap) - val backup = RestorableBackup(metadata.copy(packageMetadataMap = packageMetadataMap)) + val backup = RestorableBackup( + backupMetadata = metadata.copy(packageMetadataMap = packageMetadataMap), + repoId = repoId, + snapshot = snapshot, + ) apkRestore.installResult.test { awaitItem() // initial empty state apkRestore.restore(backup) diff --git a/app/src/test/java/com/stevesoltys/seedvault/restore/install/ApkRestoreTest.kt b/app/src/test/java/com/stevesoltys/seedvault/restore/install/ApkRestoreTest.kt index 5b4c6e0dc..215fa24eb 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/restore/install/ApkRestoreTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/restore/install/ApkRestoreTest.kt @@ -17,14 +17,14 @@ import android.graphics.drawable.Drawable import app.cash.turbine.TurbineTestContext import app.cash.turbine.test import com.stevesoltys.seedvault.BackupStateManager +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.backend.LegacyStoragePlugin import com.stevesoltys.seedvault.getRandomBase64 import com.stevesoltys.seedvault.getRandomByteArray import com.stevesoltys.seedvault.getRandomString import com.stevesoltys.seedvault.metadata.ApkSplit import com.stevesoltys.seedvault.metadata.PackageMetadata import com.stevesoltys.seedvault.metadata.PackageMetadataMap -import com.stevesoltys.seedvault.backend.LegacyStoragePlugin -import com.stevesoltys.seedvault.backend.BackendManager import com.stevesoltys.seedvault.restore.RestorableBackup import com.stevesoltys.seedvault.restore.install.ApkInstallState.FAILED import com.stevesoltys.seedvault.restore.install.ApkInstallState.FAILED_SYSTEM_APP @@ -32,7 +32,7 @@ import com.stevesoltys.seedvault.restore.install.ApkInstallState.IN_PROGRESS import com.stevesoltys.seedvault.restore.install.ApkInstallState.QUEUED import com.stevesoltys.seedvault.restore.install.ApkInstallState.SUCCEEDED import com.stevesoltys.seedvault.transport.TransportTest -import com.stevesoltys.seedvault.worker.getSignatures +import com.stevesoltys.seedvault.transport.restore.Loader import io.mockk.Runs import io.mockk.coEvery import io.mockk.every @@ -67,6 +67,7 @@ internal class ApkRestoreTest : TransportTest() { private val backupManager: IBackupManager = mockk() private val backupStateManager: BackupStateManager = mockk() private val backendManager: BackendManager = mockk() + private val loader: Loader = mockk() private val backend: Backend = mockk() private val legacyStoragePlugin: LegacyStoragePlugin = mockk() private val splitCompatChecker: ApkSplitCompatibilityChecker = mockk() @@ -78,6 +79,7 @@ internal class ApkRestoreTest : TransportTest() { backupManager = backupManager, backupStateManager = backupStateManager, backendManager = backendManager, + loader = loader, legacyStoragePlugin = legacyStoragePlugin, crypto = crypto, splitCompatChecker = splitCompatChecker, diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/CoordinatorIntegrationTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/CoordinatorIntegrationTest.kt index 7e506ce75..e7a17a4f9 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/transport/CoordinatorIntegrationTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/CoordinatorIntegrationTest.kt @@ -157,9 +157,7 @@ internal class CoordinatorIntegrationTest : TransportTest() { appData2.copyInto(value2.captured) // write the app data into the passed ByteArray appData2.size } - coEvery { - apkBackup.backupApkIfNecessary(packageInfo, any()) - } returns packageMetadata + coEvery { apkBackup.backupApkIfNecessary(packageInfo) } just Runs coEvery { backend.save(LegacyAppBackupFile.Metadata(token)) } returns metadataOutputStream @@ -238,7 +236,7 @@ internal class CoordinatorIntegrationTest : TransportTest() { appData.copyInto(value.captured) // write the app data into the passed ByteArray appData.size } - coEvery { apkBackup.backupApkIfNecessary(packageInfo, any()) } returns null + coEvery { apkBackup.backupApkIfNecessary(packageInfo) } just Runs every { settingsManager.getToken() } returns token coEvery { backend.save(LegacyAppBackupFile.Metadata(token)) @@ -307,7 +305,7 @@ internal class CoordinatorIntegrationTest : TransportTest() { } returns bOutputStream every { inputFactory.getInputStream(fileDescriptor) } returns bInputStream every { settingsManager.isQuotaUnlimited() } returns false - coEvery { apkBackup.backupApkIfNecessary(packageInfo, any()) } returns packageMetadata + coEvery { apkBackup.backupApkIfNecessary(packageInfo) } just Runs every { settingsManager.getToken() } returns token every { metadataManager.salt } returns salt coEvery { diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt index ee0e99d75..11e44a3cc 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt @@ -17,6 +17,7 @@ import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER import com.stevesoltys.seedvault.crypto.Crypto import com.stevesoltys.seedvault.getRandomBase64 import com.stevesoltys.seedvault.getRandomString +import com.stevesoltys.seedvault.header.VERSION import com.stevesoltys.seedvault.metadata.BackupMetadata import com.stevesoltys.seedvault.metadata.BackupType import com.stevesoltys.seedvault.metadata.METADATA_SALT_SIZE @@ -57,6 +58,7 @@ internal abstract class TransportTest { packageName = MAGIC_PACKAGE_MANAGER } protected val metadata = BackupMetadata( + version = VERSION, token = token, salt = getRandomBase64(METADATA_SALT_SIZE), androidVersion = Random.nextInt(), diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinatorTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinatorTest.kt index 1f9c04ac8..ef9498dae 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinatorTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinatorTest.kt @@ -14,13 +14,13 @@ import android.content.pm.PackageInfo import android.net.Uri import android.os.ParcelFileDescriptor import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER +import com.stevesoltys.seedvault.backend.BackendManager import com.stevesoltys.seedvault.coAssertThrows import com.stevesoltys.seedvault.getRandomString import com.stevesoltys.seedvault.metadata.BackupType import com.stevesoltys.seedvault.metadata.PackageMetadata import com.stevesoltys.seedvault.metadata.PackageState.NO_DATA import com.stevesoltys.seedvault.metadata.PackageState.QUOTA_EXCEEDED -import com.stevesoltys.seedvault.backend.BackendManager import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager import com.stevesoltys.seedvault.worker.ApkBackup import io.mockk.Runs @@ -273,7 +273,7 @@ internal class BackupCoordinatorTest : BackupTest() { coEvery { full.performFullBackup(packageInfo, fileDescriptor, 0, token, salt) } returns TRANSPORT_OK - coEvery { apkBackup.backupApkIfNecessary(packageInfo, any()) } returns null + coEvery { apkBackup.backupApkIfNecessary(packageInfo) } just Runs assertEquals(TRANSPORT_OK, backup.performFullBackup(packageInfo, fileDescriptor, 0)) } @@ -382,7 +382,7 @@ internal class BackupCoordinatorTest : BackupTest() { } private fun expectApkBackupAndMetadataWrite() { - coEvery { apkBackup.backupApkIfNecessary(any(), any()) } returns packageMetadata + coEvery { apkBackup.backupApkIfNecessary(packageInfo) } just Runs every { settingsManager.getToken() } returns token coEvery { backend.save(LegacyAppBackupFile.Metadata(token)) } returns metadataOutputStream every { metadataManager.onApkBackedUp(any(), packageMetadata) } just Runs diff --git a/app/src/test/java/com/stevesoltys/seedvault/worker/ApkBackupManagerTest.kt b/app/src/test/java/com/stevesoltys/seedvault/worker/ApkBackupManagerTest.kt index 246ae82de..b9487deff 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/worker/ApkBackupManagerTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/worker/ApkBackupManagerTest.kt @@ -10,11 +10,11 @@ import android.content.pm.ApplicationInfo.FLAG_ALLOW_BACKUP import android.content.pm.ApplicationInfo.FLAG_INSTALLED import android.content.pm.ApplicationInfo.FLAG_STOPPED import android.content.pm.PackageInfo +import com.stevesoltys.seedvault.backend.BackendManager import com.stevesoltys.seedvault.metadata.PackageMetadata import com.stevesoltys.seedvault.metadata.PackageState.NOT_ALLOWED import com.stevesoltys.seedvault.metadata.PackageState.UNKNOWN_ERROR import com.stevesoltys.seedvault.metadata.PackageState.WAS_STOPPED -import com.stevesoltys.seedvault.backend.BackendManager import com.stevesoltys.seedvault.transport.TransportTest import com.stevesoltys.seedvault.transport.backup.PackageService import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager @@ -31,7 +31,6 @@ import kotlinx.coroutines.runBlocking import org.calyxos.seedvault.core.backends.Backend import org.calyxos.seedvault.core.backends.LegacyAppBackupFile import org.junit.jupiter.api.Test -import java.io.ByteArrayOutputStream import java.io.IOException import java.io.OutputStream @@ -49,9 +48,8 @@ internal class ApkBackupManagerTest : TransportTest() { settingsManager = settingsManager, metadataManager = metadataManager, packageService = packageService, - apkBackup = apkBackup, iconManager = iconManager, - backendManager = backendManager, + apkBackup = apkBackup, nm = nm, ) @@ -211,17 +209,13 @@ internal class ApkBackupManagerTest : TransportTest() { nm.onApkBackup(notAllowedPackages[0].packageName, any(), 0, notAllowedPackages.size) } just Runs // no backup needed - coEvery { - apkBackup.backupApkIfNecessary(notAllowedPackages[0], any()) - } returns null + coEvery { apkBackup.backupApkIfNecessary(notAllowedPackages[0]) } just Runs // update notification for second package every { nm.onApkBackup(notAllowedPackages[1].packageName, any(), 1, notAllowedPackages.size) } just Runs // was backed up, get new packageMetadata - coEvery { - apkBackup.backupApkIfNecessary(notAllowedPackages[1], any()) - } returns packageMetadata + coEvery { apkBackup.backupApkIfNecessary(notAllowedPackages[1]) } just Runs every { metadataManager.onApkBackedUp(notAllowedPackages[1], packageMetadata) } just Runs expectFinalUpload() @@ -230,8 +224,8 @@ internal class ApkBackupManagerTest : TransportTest() { apkBackupManager.backup() coVerify { - apkBackup.backupApkIfNecessary(notAllowedPackages[0], any()) - apkBackup.backupApkIfNecessary(notAllowedPackages[1], any()) + apkBackup.backupApkIfNecessary(notAllowedPackages[0]) + apkBackup.backupApkIfNecessary(notAllowedPackages[1]) metadataOutputStream.close() } } @@ -271,10 +265,7 @@ internal class ApkBackupManagerTest : TransportTest() { } private suspend fun expectUploadIcons() { - every { settingsManager.getToken() } returns token - val stream = ByteArrayOutputStream() - coEvery { backend.save(LegacyAppBackupFile.IconsFile(token)) } returns stream - every { iconManager.uploadIcons(token, stream) } just Runs + coEvery { iconManager.uploadIcons() } just Runs } private fun expectAllAppsWillGetBackedUp() { diff --git a/app/src/test/java/com/stevesoltys/seedvault/worker/ApkBackupTest.kt b/app/src/test/java/com/stevesoltys/seedvault/worker/ApkBackupTest.kt index 2159ff32d..ce230a886 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/worker/ApkBackupTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/worker/ApkBackupTest.kt @@ -13,20 +13,25 @@ import android.content.pm.PackageInfo import android.content.pm.PackageManager import android.content.pm.Signature import android.util.PackageUtils +import com.google.protobuf.ByteString import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER import com.stevesoltys.seedvault.getRandomString -import com.stevesoltys.seedvault.metadata.ApkSplit -import com.stevesoltys.seedvault.metadata.PackageMetadata -import com.stevesoltys.seedvault.metadata.PackageState.UNKNOWN_ERROR +import com.stevesoltys.seedvault.proto.Snapshot +import com.stevesoltys.seedvault.transport.SnapshotManager +import com.stevesoltys.seedvault.transport.backup.AppBackupManager +import com.stevesoltys.seedvault.transport.backup.BackupData +import com.stevesoltys.seedvault.transport.backup.BackupReceiver import com.stevesoltys.seedvault.transport.backup.BackupTest +import com.stevesoltys.seedvault.transport.backup.SnapshotCreator +import io.mockk.Runs import io.mockk.coEvery import io.mockk.every +import io.mockk.just import io.mockk.mockk import io.mockk.mockkStatic +import io.mockk.slot import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertArrayEquals -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertThrows import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -34,34 +39,41 @@ import org.junit.jupiter.api.io.TempDir import java.io.ByteArrayOutputStream import java.io.File import java.io.IOException -import java.io.OutputStream +import java.io.InputStream import java.nio.file.Path -import kotlin.random.Random internal class ApkBackupTest : BackupTest() { private val pm: PackageManager = mockk() - private val streamGetter: suspend (name: String) -> OutputStream = mockk() + private val backupReceiver: BackupReceiver = mockk() + private val appBackupManager: AppBackupManager = mockk() + private val snapshotManager: SnapshotManager = mockk() + private val snapshotCreator: SnapshotCreator = mockk() - private val apkBackup = ApkBackup(pm, crypto, settingsManager, metadataManager) + private val apkBackup = + ApkBackup(pm, backupReceiver, appBackupManager, snapshotManager, settingsManager) private val signatureBytes = byteArrayOf(0x01, 0x02, 0x03) private val signatureHash = byteArrayOf(0x03, 0x02, 0x01) private val sigs = arrayOf(Signature(signatureBytes)) - private val packageMetadata = PackageMetadata( - time = Random.nextLong(), - version = packageInfo.longVersionCode - 1, - signatures = listOf("AwIB") - ) + private val apk = Snapshot.Apk.newBuilder() + .setVersionCode(packageInfo.longVersionCode - 1) + .addSignatures(ByteString.copyFrom(signatureHash)) + .build() + private val snapshot = Snapshot.newBuilder() + .setToken(token) + .putApps(packageInfo.packageName, Snapshot.App.newBuilder().setApk(apk).build()) + .build() init { mockkStatic(PackageUtils::class) + every { appBackupManager.snapshotCreator } returns snapshotCreator } @Test fun `does not back up @pm@`() = runBlocking { val packageInfo = PackageInfo().apply { packageName = MAGIC_PACKAGE_MANAGER } - assertNull(apkBackup.backupApkIfNecessary(packageInfo, streamGetter)) + apkBackup.backupApkIfNecessary(packageInfo) } @Test @@ -69,7 +81,7 @@ internal class ApkBackupTest : BackupTest() { every { settingsManager.backupApks() } returns false every { settingsManager.isBackupEnabled(any()) } returns true - assertNull(apkBackup.backupApkIfNecessary(packageInfo, streamGetter)) + apkBackup.backupApkIfNecessary(packageInfo) } @Test @@ -77,7 +89,7 @@ internal class ApkBackupTest : BackupTest() { every { settingsManager.backupApks() } returns true every { settingsManager.isBackupEnabled(any()) } returns false - assertNull(apkBackup.backupApkIfNecessary(packageInfo, streamGetter)) + apkBackup.backupApkIfNecessary(packageInfo) } @Test @@ -86,7 +98,7 @@ internal class ApkBackupTest : BackupTest() { every { settingsManager.isBackupEnabled(any()) } returns true every { settingsManager.backupApks() } returns true - assertNull(apkBackup.backupApkIfNecessary(packageInfo, streamGetter)) + apkBackup.backupApkIfNecessary(packageInfo) } @Test @@ -95,45 +107,50 @@ internal class ApkBackupTest : BackupTest() { every { settingsManager.isBackupEnabled(any()) } returns true every { settingsManager.backupApks() } returns true - assertNull(apkBackup.backupApkIfNecessary(packageInfo, streamGetter)) + apkBackup.backupApkIfNecessary(packageInfo) } @Test fun `does not back up the same version`() = runBlocking { packageInfo.applicationInfo.flags = FLAG_UPDATED_SYSTEM_APP - val packageMetadata = packageMetadata.copy( - version = packageInfo.longVersionCode - ) - - expectChecks(packageMetadata) + val apk = apk.toBuilder().setVersionCode(packageInfo.longVersionCode).build() + val app = Snapshot.App.newBuilder().setApk(apk).build() + expectChecks(snapshot.toBuilder().putApps(packageInfo.packageName, app).build()) - assertNull(apkBackup.backupApkIfNecessary(packageInfo, streamGetter)) + apkBackup.backupApkIfNecessary(packageInfo) } @Test fun `does back up the same version when signatures changes`() { packageInfo.applicationInfo.sourceDir = "/tmp/doesNotExist" - - expectChecks() + val apk = apk.toBuilder() + .clearSignatures() + .addSignatures(ByteString.copyFromUtf8("foo")) + .setVersionCode(packageInfo.longVersionCode) + .build() + val app = Snapshot.App.newBuilder().setApk(apk).build() + expectChecks(snapshot.toBuilder().putApps(packageInfo.packageName, app).build()) + every { + pm.getInstallSourceInfo(packageInfo.packageName) + } returns InstallSourceInfo(null, null, null, getRandomString()) assertThrows(IOException::class.java) { runBlocking { - assertNull(apkBackup.backupApkIfNecessary(packageInfo, streamGetter)) + apkBackup.backupApkIfNecessary(packageInfo) } } + Unit } @Test fun `do not accept empty signature`() = runBlocking { every { settingsManager.backupApks() } returns true every { settingsManager.isBackupEnabled(any()) } returns true - every { - metadataManager.getPackageMetadata(packageInfo.packageName) - } returns packageMetadata + every { snapshotManager.latestSnapshot } returns snapshot every { sigInfo.hasMultipleSigners() } returns false every { sigInfo.signingCertificateHistory } returns emptyArray() - assertNull(apkBackup.backupApkIfNecessary(packageInfo, streamGetter)) + apkBackup.backupApkIfNecessary(packageInfo) } @Test @@ -145,27 +162,24 @@ internal class ApkBackupTest : BackupTest() { writeBytes(apkBytes) }.absolutePath val apkOutputStream = ByteArrayOutputStream() - val updatedMetadata = PackageMetadata( - time = packageMetadata.time, - state = UNKNOWN_ERROR, - version = packageInfo.longVersionCode, - installer = getRandomString(), - sha256 = "eHx5jjmlvBkQNVuubQzYejay4Q_QICqD47trAF2oNHI", - signatures = packageMetadata.signatures - ) + val installer = getRandomString() + val capturedStream = slot() expectChecks() - every { metadataManager.salt } returns salt - every { crypto.getNameForApk(salt, packageInfo.packageName) } returns name - coEvery { streamGetter.invoke(name) } returns apkOutputStream every { pm.getInstallSourceInfo(packageInfo.packageName) - } returns InstallSourceInfo(null, null, null, updatedMetadata.installer) + } returns InstallSourceInfo(null, null, null, installer) + coEvery { backupReceiver.readFromStream(capture(capturedStream)) } answers { + capturedStream.captured.copyTo(apkOutputStream) + } + coEvery { backupReceiver.finalize() } returns BackupData(emptyList(), emptyMap()) + every { + snapshotCreator.onApkBackedUp(packageInfo.packageName, match { + it.installer == installer + }, emptyMap()) + } just Runs - assertEquals( - updatedMetadata, - apkBackup.backupApkIfNecessary(packageInfo, streamGetter) - ) + apkBackup.backupApkIfNecessary(packageInfo) assertArrayEquals(apkBytes, apkOutputStream.toByteArray()) } @@ -184,9 +198,7 @@ internal class ApkBackupTest : BackupTest() { packageInfo.splitNames = arrayOf(split1Name, split2Name) // create two split APKs val split1Bytes = byteArrayOf(0x07, 0x08, 0x09) - val split1Sha256 = "ZqZ1cVH47lXbEncWx-Pc4L6AdLZOIO2lQuXB5GypxB4" val split2Bytes = byteArrayOf(0x01, 0x02, 0x03) - val split2Sha256 = "A5BYxvLAy0ksUzsKTRTvd8wPeKvMztUofYShogEc-4E" packageInfo.applicationInfo.splitSourceDirs = arrayOf( File(tmpFile, "test-$split1Name.apk").apply { assertTrue(createNewFile()) @@ -201,54 +213,39 @@ internal class ApkBackupTest : BackupTest() { val apkOutputStream = ByteArrayOutputStream() val split1OutputStream = ByteArrayOutputStream() val split2OutputStream = ByteArrayOutputStream() - // expected new metadata for package - val updatedMetadata = PackageMetadata( - time = packageMetadata.time, - state = UNKNOWN_ERROR, - version = packageInfo.longVersionCode, - installer = getRandomString(), - splits = listOf( - ApkSplit(split1Name, split1Bytes.size.toLong(), split1Sha256), - ApkSplit(split2Name, split2Bytes.size.toLong(), split2Sha256) - ), - sha256 = "eHx5jjmlvBkQNVuubQzYejay4Q_QICqD47trAF2oNHI", - signatures = packageMetadata.signatures - ) - val suffixName1 = getRandomString() - val suffixName2 = getRandomString() + val capturedStream = slot() + val installer = getRandomString() expectChecks() - every { metadataManager.salt } returns salt - every { crypto.getNameForApk(salt, packageInfo.packageName) } returns name - every { - crypto.getNameForApk(salt, packageInfo.packageName, split1Name) - } returns suffixName1 - every { - crypto.getNameForApk(salt, packageInfo.packageName, split2Name) - } returns suffixName2 - coEvery { streamGetter.invoke(name) } returns apkOutputStream - coEvery { streamGetter.invoke(suffixName1) } returns split1OutputStream - coEvery { streamGetter.invoke(suffixName2) } returns split2OutputStream - every { pm.getInstallSourceInfo(packageInfo.packageName) - } returns InstallSourceInfo(null, null, null, updatedMetadata.installer) - - assertEquals( - updatedMetadata, - apkBackup.backupApkIfNecessary(packageInfo, streamGetter) - ) + } returns InstallSourceInfo(null, null, null, installer) + coEvery { backupReceiver.readFromStream(capture(capturedStream)) } answers { + capturedStream.captured.copyTo(apkOutputStream) + } andThenAnswer { + capturedStream.captured.copyTo(split1OutputStream) + } andThenAnswer { + capturedStream.captured.copyTo(split2OutputStream) + } + coEvery { backupReceiver.finalize() } returns BackupData(emptyList(), emptyMap()) + every { + snapshotCreator.onApkBackedUp(packageInfo.packageName, match { + it.installer == installer && + it.getSplits(1).name == split1Name && + it.getSplits(2).name == split2Name + }, emptyMap()) + } just Runs + + apkBackup.backupApkIfNecessary(packageInfo) assertArrayEquals(apkBytes, apkOutputStream.toByteArray()) assertArrayEquals(split1Bytes, split1OutputStream.toByteArray()) assertArrayEquals(split2Bytes, split2OutputStream.toByteArray()) } - private fun expectChecks(packageMetadata: PackageMetadata = this.packageMetadata) { + private fun expectChecks(snapshot: Snapshot = this.snapshot) { every { settingsManager.isBackupEnabled(any()) } returns true every { settingsManager.backupApks() } returns true - every { - metadataManager.getPackageMetadata(packageInfo.packageName) - } returns packageMetadata + every { snapshotManager.latestSnapshot } returns snapshot every { PackageUtils.computeSha256DigestBytes(signatureBytes) } returns signatureHash every { sigInfo.hasMultipleSigners() } returns false every { sigInfo.signingCertificateHistory } returns sigs diff --git a/libs/Android.bp b/libs/Android.bp index 36158313a..ae63f75ad 100644 --- a/libs/Android.bp +++ b/libs/Android.bp @@ -16,6 +16,12 @@ java_import { sdk_version: "current", } +java_import { + name: "seedvault-lib-chunker", + jars: ["seedvault-chunker-0.1.jar"], + sdk_version: "current", +} + java_import { name: "seedvault-lib-kotlin-logging-jvm", jars: ["kotlin-logging-jvm-6.0.3.jar"], diff --git a/libs/seedvault-chunker-0.1.jar b/libs/seedvault-chunker-0.1.jar new file mode 100644 index 000000000..a452dfe52 Binary files /dev/null and b/libs/seedvault-chunker-0.1.jar differ