Skip to content

Commit

Permalink
Clean up metadata as it lost most of its importance
Browse files Browse the repository at this point in the history 
Historically, metadata was uploaded to the backend after each app update and contained all essential data that is now in snapshots. We still support reading metadata for legacy backups and use the metadata classes as a common wrapper for snapshots. However, there is no need anymore to write out complete historic metadata and maintain duplicated unused information there. This got removed. THe information we do still save and write out is only for UI representation of backup state.

The time of last backup is now managed by SettingsManager.
  • Loading branch information
@grote
grote committed Sep 13, 2024
1 parent 15a366d commit 32738fa
Show file tree
Hide file tree
Showing 20 changed files with 76 additions and 772 deletions.
1 change: 0 additions & 1 deletion app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,6 @@ internal interface LargeTestBase : KoinComponent {

fun resetApplicationState() {
backupManager.setAutoRestore(false)
settingsManager.token = null

val sharedPreferences = permitDiskReads {
PreferenceManager.getDefaultSharedPreferences(targetContext)
Expand Down
9 changes: 4 additions & 5 deletions app/src/main/java/com/stevesoltys/seedvault/UsbIntentReceiver.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import android.os.Looper
import android.provider.DocumentsContract
import android.util.Log
import androidx.core.content.ContextCompat.startForegroundService
import com.stevesoltys.seedvault.metadata.MetadataManager
import com.stevesoltys.seedvault.settings.FlashDrive
import com.stevesoltys.seedvault.settings.SettingsManager
import com.stevesoltys.seedvault.storage.StorageBackupService
Expand All @@ -37,7 +36,6 @@ class UsbIntentReceiver : UsbMonitor() {

// using KoinComponent would crash robolectric tests :(
private val settingsManager: SettingsManager by lazy { get().get() }
private val metadataManager: MetadataManager by lazy { get().get() }
private val backupManager: IBackupManager by lazy { get().get() }

override fun shouldMonitorStatus(context: Context, action: String, device: UsbDevice): Boolean {
Expand All @@ -47,14 +45,15 @@ class UsbIntentReceiver : UsbMonitor() {
val attachedFlashDrive = FlashDrive.from(device)
return if (savedFlashDrive == attachedFlashDrive) {
Log.d(TAG, "Matches stored device, checking backup time...")
val backupMillis = System.currentTimeMillis() - metadataManager.getLastBackupTime()
val lastBackupTime = settingsManager.lastBackupTime.value ?: 0
val backupMillis = System.currentTimeMillis() - lastBackupTime
if (backupMillis >= settingsManager.backupFrequencyInMillis) {
Log.d(TAG, "Last backup older than it should be, requesting a backup...")
Log.d(TAG, " ${Date(metadataManager.getLastBackupTime())}")
Log.d(TAG, " ${Date(lastBackupTime)}")
true
} else {
Log.d(TAG, "We have a recent backup, not requesting a new one.")
Log.d(TAG, " ${Date(metadataManager.getLastBackupTime())}")
Log.d(TAG, " ${Date(lastBackupTime)}")
false
}
} else {
Expand Down
84 changes: 7 additions & 77 deletions app/src/main/java/com/stevesoltys/seedvault/metadata/MetadataManager.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,8 @@ import android.content.pm.PackageInfo
import android.util.Log
import androidx.annotation.VisibleForTesting
import androidx.annotation.WorkerThread
import androidx.lifecycle.LiveData
import androidx.lifecycle.MutableLiveData
import androidx.lifecycle.distinctUntilChanged
import com.stevesoltys.seedvault.Clock
import com.stevesoltys.seedvault.header.VERSION
import com.stevesoltys.seedvault.metadata.PackageState.APK_AND_DATA
import com.stevesoltys.seedvault.settings.SettingsManager
import com.stevesoltys.seedvault.transport.backup.PackageService
import com.stevesoltys.seedvault.transport.backup.isSystemApp
import java.io.FileNotFoundException
Expand All @@ -37,7 +32,6 @@ internal class MetadataManager(
private val metadataWriter: MetadataWriter,
private val metadataReader: MetadataReader,
private val packageService: PackageService,
private val settingsManager: SettingsManager,
) {

private val uninitializedMetadata = BackupMetadata(token = -42L, salt = "foo bar")
Expand All @@ -54,7 +48,6 @@ internal class MetadataManager(
// This should cause requiresInit() return true
uninitializedMetadata.copy(version = (-1).toByte())
}
mLastBackupTime.postValue(field.time)
}
return field
}
Expand All @@ -63,40 +56,6 @@ internal class MetadataManager(
packageService.launchableSystemApps.map { it.activityInfo.packageName }.toSet()
}

/**
* Call this after a package's APK has been backed up successfully.
*
* It updates the packages' metadata to the internal cache.
*/
@Synchronized
@Throws(IOException::class)
fun onApkBackedUp(
packageInfo: PackageInfo,
packageMetadata: PackageMetadata,
) {
val packageName = packageInfo.packageName
metadata.packageMetadataMap[packageName]?.let {
check(packageMetadata.version != null) {
"APK backup returned version null"
}
}
val oldPackageMetadata = metadata.packageMetadataMap[packageName]
?: PackageMetadata()
modifyCachedMetadata {
val isSystemApp = packageInfo.isSystemApp()
metadata.packageMetadataMap[packageName] = oldPackageMetadata.copy(
name = packageInfo.applicationInfo?.loadLabel(context.packageManager),
system = isSystemApp,
isLaunchableSystemApp = isSystemApp && launchableSystemApps.contains(packageName),
version = packageMetadata.version,
installer = packageMetadata.installer,
splits = packageMetadata.splits,
sha256 = packageMetadata.sha256,
signatures = packageMetadata.signatures
)
}
}

/**
* Call this after a package has been backed up successfully.
*
Expand All @@ -115,16 +74,13 @@ internal class MetadataManager(
val packageName = packageInfo.packageName
modifyCachedMetadata {
val now = clock.time()
metadata.time = now
metadata.d2dBackup = settingsManager.d2dBackupsEnabled()
metadata.packageMetadataMap.getOrPut(packageName) {
val isSystemApp = packageInfo.isSystemApp()
PackageMetadata(
time = now,
state = APK_AND_DATA,
backupType = type,
size = size,
name = packageInfo.applicationInfo?.loadLabel(context.packageManager),
system = isSystemApp,
isLaunchableSystemApp = isSystemApp &&
launchableSystemApps.contains(packageName),
Expand All @@ -135,10 +91,6 @@ internal class MetadataManager(
backupType = type
// don't override a previous K/V size, if there were no K/V changes
if (size != null) this.size = size
// update name, if none was set, yet (can happen while migrating to storing names)
if (this.name == null) {
this.name = packageInfo.applicationInfo?.loadLabel(context.packageManager)
}
}
}
}
Expand Down Expand Up @@ -203,9 +155,15 @@ internal class MetadataManager(
}
}

@Synchronized
fun getPackageMetadata(packageName: String): PackageMetadata? {
return metadata.packageMetadataMap[packageName]?.copy()
}

@Throws(IOException::class)
private fun modifyCachedMetadata(modFun: () -> Unit) {
val oldMetadata = metadata.copy( // copy map, otherwise it will re-use same reference
val oldMetadata = metadata.copy(
// copy map, otherwise it will re-use same reference
packageMetadataMap = PackageMetadataMap(metadata.packageMetadataMap),
)
try {
Expand All @@ -217,34 +175,6 @@ internal class MetadataManager(
metadata = oldMetadata
throw IOException(e)
}
mLastBackupTime.postValue(metadata.time) // TODO only do after snapshot was written
}

/**
* Returns the last backup time in unix epoch milli seconds.
*
* Note that this might be a blocking I/O call.
*/
@Synchronized
fun getLastBackupTime(): Long = mLastBackupTime.value ?: metadata.time

private val mLastBackupTime = MutableLiveData<Long>()
internal val lastBackupTime: LiveData<Long> = mLastBackupTime.distinctUntilChanged()

internal val salt: String
@Synchronized get() = metadata.salt

internal val requiresInit: Boolean
@Synchronized get() = metadata == uninitializedMetadata || metadata.version < VERSION

@Synchronized
fun getPackageMetadata(packageName: String): PackageMetadata? {
return metadata.packageMetadataMap[packageName]?.copy()
}

@Synchronized
fun getPackagesBackupSize(): Long {
return metadata.packageMetadataMap.values.sumOf { it.size ?: 0L }
}

@Synchronized
Expand Down
4 changes: 2 additions & 2 deletions app/src/main/java/com/stevesoltys/seedvault/metadata/MetadataModule.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import org.koin.android.ext.koin.androidContext
import org.koin.dsl.module

val metadataModule = module {
single { MetadataManager(androidContext(), get(), get(), get(), get(), get()) }
single<MetadataWriter> { MetadataWriterImpl(get()) }
single { MetadataManager(androidContext(), get(), get(), get(), get()) }
single<MetadataWriter> { MetadataWriterImpl() }
single<MetadataReader> { MetadataReaderImpl(get()) }
}
45 changes: 2 additions & 43 deletions app/src/main/java/com/stevesoltys/seedvault/metadata/MetadataWriter.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,42 +6,18 @@
package com.stevesoltys.seedvault.metadata

import com.stevesoltys.seedvault.Utf8
import com.stevesoltys.seedvault.crypto.Crypto
import com.stevesoltys.seedvault.metadata.PackageState.APK_AND_DATA
import org.json.JSONArray
import org.json.JSONObject
import java.io.IOException
import java.io.OutputStream

interface MetadataWriter {
@Throws(IOException::class)
fun write(metadata: BackupMetadata, outputStream: OutputStream)

fun encode(metadata: BackupMetadata): ByteArray
}

internal class MetadataWriterImpl(private val crypto: Crypto) : MetadataWriter {

@Throws(IOException::class)
override fun write(metadata: BackupMetadata, outputStream: OutputStream) {
outputStream.write(ByteArray(1).apply { this[0] = metadata.version })
crypto.newEncryptingStreamV1(outputStream, getAD(metadata.version, metadata.token)).use {
it.write(encode(metadata))
}
}
internal class MetadataWriterImpl : MetadataWriter {

override fun encode(metadata: BackupMetadata): ByteArray {
val json = JSONObject().apply {
put(JSON_METADATA, JSONObject().apply {
put(JSON_METADATA_VERSION, metadata.version.toInt())
put(JSON_METADATA_TOKEN, metadata.token)
put(JSON_METADATA_SALT, metadata.salt)
put(JSON_METADATA_TIME, metadata.time)
put(JSON_METADATA_SDK_INT, metadata.androidVersion)
put(JSON_METADATA_INCREMENTAL, metadata.androidIncremental)
put(JSON_METADATA_NAME, metadata.deviceName)
put(JSON_METADATA_D2D_BACKUP, metadata.d2dBackup)
})
put(JSON_METADATA, JSONObject())
}
for ((packageName, packageMetadata) in metadata.packageMetadataMap) {
json.put(packageName, JSONObject().apply {
Expand All @@ -57,31 +33,14 @@ internal class MetadataWriterImpl(private val crypto: Crypto) : MetadataWriter {
if (packageMetadata.size != null) {
put(JSON_PACKAGE_SIZE, packageMetadata.size)
}
if (packageMetadata.name != null) {
put(JSON_PACKAGE_APP_NAME, packageMetadata.name)
}
if (packageMetadata.system) {
put(JSON_PACKAGE_SYSTEM, true)
}
if (packageMetadata.isLaunchableSystemApp) {
put(JSON_PACKAGE_SYSTEM_LAUNCHER, true)
}
packageMetadata.version?.let { put(JSON_PACKAGE_VERSION, it) }
packageMetadata.installer?.let { put(JSON_PACKAGE_INSTALLER, it) }
packageMetadata.splits?.let { splits ->
put(JSON_PACKAGE_SPLITS, JSONArray().apply {
for (split in splits) put(JSONObject().apply {
put(JSON_PACKAGE_SPLIT_NAME, split.name)
if (split.size != null) put(JSON_PACKAGE_SIZE, split.size)
put(JSON_PACKAGE_SHA256, split.sha256)
})
})
}
packageMetadata.sha256?.let { put(JSON_PACKAGE_SHA256, it) }
packageMetadata.signatures?.let { put(JSON_PACKAGE_SIGNATURES, JSONArray(it)) }
})
}
return json.toString().toByteArray(Utf8)
}

}
32 changes: 24 additions & 8 deletions app/src/main/java/com/stevesoltys/seedvault/settings/SettingsManager.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import android.content.SharedPreferences.OnSharedPreferenceChangeListener
import android.hardware.usb.UsbDevice
import android.net.Uri
import androidx.annotation.UiThread
import androidx.lifecycle.LiveData
import androidx.lifecycle.MutableLiveData
import androidx.preference.PreferenceManager
import com.stevesoltys.seedvault.backend.webdav.WebDavHandler.Companion.createWebDavProperties
import com.stevesoltys.seedvault.permitDiskReads
Expand Down Expand Up @@ -55,20 +57,19 @@ private const val PREF_KEY_BACKUP_APP_BLACKLIST = "backupAppBlacklist"
private const val PREF_KEY_BACKUP_STORAGE = "backup_storage"
internal const val PREF_KEY_UNLIMITED_QUOTA = "unlimited_quota"
internal const val PREF_KEY_D2D_BACKUPS = "d2d_backups"
internal const val PREF_KEY_LAST_BACKUP = "lastBackup"

class SettingsManager(private val context: Context) {

private val prefs = permitDiskReads {
PreferenceManager.getDefaultSharedPreferences(context)
}
private val mLastBackupTime = MutableLiveData(prefs.getLong(PREF_KEY_LAST_BACKUP, -1))

fun registerOnSharedPreferenceChangeListener(listener: OnSharedPreferenceChangeListener) {
prefs.registerOnSharedPreferenceChangeListener(listener)
}

fun unregisterOnSharedPreferenceChangeListener(listener: OnSharedPreferenceChangeListener) {
prefs.unregisterOnSharedPreferenceChangeListener(listener)
}
/**
* Returns a LiveData of the last backup time in unix epoch milli seconds.
*/
internal val lastBackupTime: LiveData<Long> = mLastBackupTime

/**
* This gets accessed by non-UI threads when saving with [PreferenceManager]
Expand All @@ -81,7 +82,7 @@ class SettingsManager(private val context: Context) {

@Volatile
var token: Long? = null
set(newToken) {
private set(newToken) {
if (newToken == null) {
prefs.edit()
.remove(PREF_KEY_TOKEN)
Expand Down Expand Up @@ -121,6 +122,21 @@ class SettingsManager(private val context: Context) {
}
}

fun registerOnSharedPreferenceChangeListener(listener: OnSharedPreferenceChangeListener) {
prefs.registerOnSharedPreferenceChangeListener(listener)
}

fun unregisterOnSharedPreferenceChangeListener(listener: OnSharedPreferenceChangeListener) {
prefs.unregisterOnSharedPreferenceChangeListener(listener)
}

fun onSuccessfulBackupCompleted(token: Long) {
this.token = token
val now = System.currentTimeMillis()
prefs.edit().putLong(PREF_KEY_LAST_BACKUP, now).apply()
mLastBackupTime.postValue(now)
}

fun setStorageBackend(plugin: Backend) {
val value = when (plugin) {
is SafBackend -> StoragePluginType.SAF
Expand Down
19 changes: 1 addition & 18 deletions app/src/main/java/com/stevesoltys/seedvault/settings/SettingsViewModel.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ internal class SettingsViewModel(
private val mBackupPossible = MutableLiveData(false)
val backupPossible: LiveData<Boolean> = mBackupPossible

internal val lastBackupTime = metadataManager.lastBackupTime
internal val lastBackupTime = settingsManager.lastBackupTime
internal val appBackupWorkInfo =
workManager.getWorkInfosForUniqueWorkLiveData(UNIQUE_WORK_NAME).map {
it.getOrNull(0)
Expand Down Expand Up @@ -146,8 +146,6 @@ internal class SettingsViewModel(
initialValue = false,
)
scope.launch {
// ensures the lastBackupTime LiveData gets set
metadataManager.getLastBackupTime()
// update running state
isBackupRunning.collect {
onBackupRunningStateChanged()
Expand Down Expand Up @@ -270,21 +268,6 @@ internal class SettingsViewModel(

fun onBackupEnabled(enabled: Boolean) {
if (enabled) {
if (metadataManager.requiresInit) {
val onError: () -> Unit = {
viewModelScope.launch(Dispatchers.Main) {
val res = R.string.storage_check_fragment_backup_error
Toast.makeText(app, res, LENGTH_LONG).show()
}
}
viewModelScope.launch(Dispatchers.IO) {
backupInitializer.initialize(onError) {
mInitEvent.postEvent(false)
scheduleAppBackup(CANCEL_AND_REENQUEUE)
}
mInitEvent.postEvent(true)
}
}
// enable call log backups for existing installs (added end of 2020)
enableCallLogBackup()
} else {
Expand Down
Loading

0 comments on commit 32738fa

Please sign in to comment.