diff --git a/.changeset/database-migration-feature.md b/.changeset/database-migration-feature.md new file mode 100644 index 000000000..b44073ab3 --- /dev/null +++ b/.changeset/database-migration-feature.md @@ -0,0 +1,15 @@ +--- +"scopes": minor +--- + +Add automatic database migration system for SQLDelight databases + +- Implement DatabaseMigrationManager with thread-safe migration execution +- Add version tracking using SQLite PRAGMA user_version +- Support for custom migration callbacks at specific versions +- Automatic schema creation for fresh databases +- Fail-fast behavior when database version is newer than application +- Database-level locking to prevent concurrent migrations across processes +- Proper resource management and exception handling +- Migration files (.sqm) for all bounded contexts (scope-management, event-store, device-synchronization) +- Comprehensive documentation and examples in database-migration-guide.md \ No newline at end of file diff --git a/contexts/device-synchronization/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/devicesync/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt b/contexts/device-synchronization/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/devicesync/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt index 1d36c9623..a3dac61bb 100644 --- a/contexts/device-synchronization/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/devicesync/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt +++ b/contexts/device-synchronization/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/devicesync/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt @@ -3,20 +3,31 @@ package io.github.kamiazya.scopes.devicesync.infrastructure.sqldelight import app.cash.sqldelight.db.SqlDriver import app.cash.sqldelight.driver.jdbc.sqlite.JdbcSqliteDriver import io.github.kamiazya.scopes.devicesync.db.DeviceSyncDatabase +import io.github.kamiazya.scopes.platform.infrastructure.database.DatabaseMigrationManager +import io.github.kamiazya.scopes.platform.infrastructure.version.ApplicationVersion /** * Provides SQLDelight database instances for Device Synchronization. + * + * This provider creates databases with migration support. */ object SqlDelightDatabaseProvider { + private val migrationManager = DatabaseMigrationManager.createDefault() + /** * Creates a new DeviceSyncDatabase instance. + * Automatically handles schema creation and migration based on version differences. */ fun createDatabase(databasePath: String): DeviceSyncDatabase { val driver: SqlDriver = JdbcSqliteDriver("jdbc:sqlite:$databasePath") - // Create the database schema - DeviceSyncDatabase.Schema.create(driver) + // Perform migration if needed + migrationManager.migrate( + driver = driver, + schema = DeviceSyncDatabase.Schema, + targetVersion = ApplicationVersion.SchemaVersions.DEVICE_SYNCHRONIZATION + ) // Enable foreign keys driver.execute(null, "PRAGMA foreign_keys=ON", 0) @@ -26,10 +37,21 @@ object SqlDelightDatabaseProvider { /** * Creates an in-memory database for testing. + * Always creates a fresh schema without migration. */ fun createInMemoryDatabase(): DeviceSyncDatabase { val driver: SqlDriver = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) + + // For in-memory databases, always create fresh schema DeviceSyncDatabase.Schema.create(driver) + + // Set the version for consistency + driver.execute( + identifier = null, + sql = "PRAGMA user_version = ${ApplicationVersion.SchemaVersions.DEVICE_SYNCHRONIZATION}", + parameters = 0 + ) + return DeviceSyncDatabase(driver) } } diff --git a/contexts/device-synchronization/infrastructure/src/main/sqldelight/migrations/1.sqm b/contexts/device-synchronization/infrastructure/src/main/sqldelight/migrations/1.sqm new file mode 100644 index 000000000..89d51a3dd --- /dev/null +++ b/contexts/device-synchronization/infrastructure/src/main/sqldelight/migrations/1.sqm @@ -0,0 +1,23 @@ +-- Migration from version 1 to version 2 +-- This is a placeholder migration file for the Device Synchronization context +-- +-- Example migrations for device sync (uncomment when needed): +-- +-- Add new column for sync status +-- ALTER TABLE devices ADD COLUMN last_error TEXT; +-- +-- Add table for sync conflicts +-- CREATE TABLE IF NOT EXISTS sync_conflicts ( +-- id TEXT PRIMARY KEY NOT NULL, +-- device_id TEXT NOT NULL, +-- conflict_data TEXT NOT NULL, +-- created_at INTEGER NOT NULL, +-- resolved_at INTEGER, +-- FOREIGN KEY (device_id) REFERENCES devices(id) ON DELETE CASCADE +-- ); +-- +-- Note: This file is currently a placeholder. +-- When you need to make schema changes in the future: +-- 1. Uncomment and modify the SQL statements above +-- 2. Update ApplicationVersion.SchemaVersions.DEVICE_SYNCHRONIZATION to 2 +-- 3. Add a new VersionMapping in ApplicationVersion.versionHistory \ No newline at end of file diff --git a/contexts/event-store/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/eventstore/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt b/contexts/event-store/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/eventstore/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt index e91ff0a61..b20412b5d 100644 --- a/contexts/event-store/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/eventstore/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt +++ b/contexts/event-store/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/eventstore/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt @@ -3,20 +3,31 @@ package io.github.kamiazya.scopes.eventstore.infrastructure.sqldelight import app.cash.sqldelight.db.SqlDriver import app.cash.sqldelight.driver.jdbc.sqlite.JdbcSqliteDriver import io.github.kamiazya.scopes.eventstore.db.EventStoreDatabase +import io.github.kamiazya.scopes.platform.infrastructure.database.DatabaseMigrationManager +import io.github.kamiazya.scopes.platform.infrastructure.version.ApplicationVersion /** * Provides SQLDelight database instances for Event Store. + * + * This provider creates databases with migration support. */ object SqlDelightDatabaseProvider { + private val migrationManager = DatabaseMigrationManager.createDefault() + /** * Creates a new EventStoreDatabase instance. + * Automatically handles schema creation and migration based on version differences. */ fun createDatabase(databasePath: String): EventStoreDatabase { val driver: SqlDriver = JdbcSqliteDriver("jdbc:sqlite:$databasePath") - // Create the database schema - EventStoreDatabase.Schema.create(driver) + // Perform migration if needed + migrationManager.migrate( + driver = driver, + schema = EventStoreDatabase.Schema, + targetVersion = ApplicationVersion.SchemaVersions.EVENT_STORE + ) // Enable foreign keys driver.execute(null, "PRAGMA foreign_keys=ON", 0) @@ -26,10 +37,21 @@ object SqlDelightDatabaseProvider { /** * Creates an in-memory database for testing. + * Always creates a fresh schema without migration. */ fun createInMemoryDatabase(): EventStoreDatabase { val driver: SqlDriver = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) + + // For in-memory databases, always create fresh schema EventStoreDatabase.Schema.create(driver) + + // Set the version for consistency + driver.execute( + identifier = null, + sql = "PRAGMA user_version = ${ApplicationVersion.SchemaVersions.EVENT_STORE}", + parameters = 0 + ) + return EventStoreDatabase(driver) } } diff --git a/contexts/event-store/infrastructure/src/main/sqldelight/migrations/1.sqm b/contexts/event-store/infrastructure/src/main/sqldelight/migrations/1.sqm new file mode 100644 index 000000000..638e675f4 --- /dev/null +++ b/contexts/event-store/infrastructure/src/main/sqldelight/migrations/1.sqm @@ -0,0 +1,16 @@ +-- Migration from version 1 to version 2 +-- This is a placeholder migration file for the Event Store context +-- +-- Example migrations for event store (uncomment when needed): +-- +-- Add new column for event metadata +-- ALTER TABLE events ADD COLUMN correlation_id TEXT; +-- +-- Add index for correlation tracking +-- CREATE INDEX IF NOT EXISTS idx_events_correlation ON events(correlation_id); +-- +-- Note: This file is currently a placeholder. +-- When you need to make schema changes in the future: +-- 1. Uncomment and modify the SQL statements above +-- 2. Update ApplicationVersion.SchemaVersions.EVENT_STORE to 2 +-- 3. Add a new VersionMapping in ApplicationVersion.versionHistory \ No newline at end of file diff --git a/contexts/scope-management/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/scopemanagement/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt b/contexts/scope-management/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/scopemanagement/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt index 25c0d2b14..c55241044 100644 --- a/contexts/scope-management/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/scopemanagement/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt +++ b/contexts/scope-management/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/scopemanagement/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt @@ -1,48 +1,74 @@ package io.github.kamiazya.scopes.scopemanagement.infrastructure.sqldelight +import io.github.kamiazya.scopes.platform.infrastructure.database.DatabaseMigrationManager import io.github.kamiazya.scopes.platform.infrastructure.database.ManagedSqlDriver +import io.github.kamiazya.scopes.platform.infrastructure.version.ApplicationVersion import io.github.kamiazya.scopes.scopemanagement.db.ScopeManagementDatabase /** * Provides SQLDelight database instances for Scope Management. * - * This provider creates databases with automatic resource management. + * This provider creates databases with automatic resource management and migration support. * The returned ManagedDatabase wrapper ensures proper cleanup on close. */ object SqlDelightDatabaseProvider { + private val migrationManager = DatabaseMigrationManager.createDefault() + /** * Wrapper that combines database and its managed driver for proper cleanup. + * Implements AutoCloseable to ensure resources are properly released. */ - class ManagedDatabase(private val database: ScopeManagementDatabase, private val managedDriver: AutoCloseable) : - ScopeManagementDatabase by database, - AutoCloseable { + class ManagedDatabase( + private val database: ScopeManagementDatabase, + private val managedDriver: AutoCloseable + ) : ScopeManagementDatabase by database, AutoCloseable { override fun close() { + // Close the driver to release file handles and WAL locks managedDriver.close() } } /** * Creates a new ScopeManagementDatabase instance with automatic resource management. + * Automatically handles schema creation and migration based on version differences. + * + * @return ManagedDatabase that must be closed when no longer needed */ - fun createDatabase(databasePath: String): ScopeManagementDatabase { + fun createDatabase(databasePath: String): ManagedDatabase { val managedDriver = ManagedSqlDriver.createWithDefaults(databasePath) val driver = managedDriver.driver - // Create the database schema - ScopeManagementDatabase.Schema.create(driver) + // Perform migration if needed + migrationManager.migrate( + driver = driver, + schema = ScopeManagementDatabase.Schema, + targetVersion = ApplicationVersion.SchemaVersions.SCOPE_MANAGEMENT + ) return ManagedDatabase(ScopeManagementDatabase(driver), managedDriver) } /** * Creates an in-memory database for testing. + * Always creates a fresh schema without migration. + * + * @return ManagedDatabase that must be closed when no longer needed */ - fun createInMemoryDatabase(): ScopeManagementDatabase { + fun createInMemoryDatabase(): ManagedDatabase { val managedDriver = ManagedSqlDriver(":memory:") val driver = managedDriver.driver + // For in-memory databases, always create fresh schema ScopeManagementDatabase.Schema.create(driver) + + // Set the version for consistency + driver.execute( + identifier = null, + sql = "PRAGMA user_version = ${ApplicationVersion.SchemaVersions.SCOPE_MANAGEMENT}", + parameters = 0 + ) + return ManagedDatabase(ScopeManagementDatabase(driver), managedDriver) } } diff --git a/contexts/scope-management/infrastructure/src/main/sqldelight/migrations/1.sqm b/contexts/scope-management/infrastructure/src/main/sqldelight/migrations/1.sqm new file mode 100644 index 000000000..ac3c406ad --- /dev/null +++ b/contexts/scope-management/infrastructure/src/main/sqldelight/migrations/1.sqm @@ -0,0 +1,25 @@ +-- Migration from version 1 to version 2 +-- This is an example migration file that will be used when upgrading from schema version 1 to 2 +-- +-- Example migrations (uncomment and modify as needed for actual schema changes): +-- +-- Add a new column to the scopes table +-- ALTER TABLE scopes ADD COLUMN status TEXT DEFAULT 'active'; +-- +-- Create a new index +-- CREATE INDEX IF NOT EXISTS idx_scopes_status ON scopes(status); +-- +-- Add a new table +-- CREATE TABLE IF NOT EXISTS scope_tags ( +-- scope_id TEXT NOT NULL, +-- tag TEXT NOT NULL, +-- created_at INTEGER NOT NULL, +-- PRIMARY KEY (scope_id, tag), +-- FOREIGN KEY (scope_id) REFERENCES scopes(id) ON DELETE CASCADE +-- ); +-- +-- Note: This file is currently a placeholder. +-- When you need to make schema changes in the future: +-- 1. Uncomment and modify the SQL statements above +-- 2. Update ApplicationVersion.SchemaVersions.SCOPE_MANAGEMENT to 2 +-- 3. Add a new VersionMapping in ApplicationVersion.versionHistory \ No newline at end of file diff --git a/docs/guides/development/database-migration-guide.md b/docs/guides/development/database-migration-guide.md new file mode 100644 index 000000000..1fce6a7c3 --- /dev/null +++ b/docs/guides/development/database-migration-guide.md @@ -0,0 +1,274 @@ +# Database Migration Guide + +## Overview + +This guide explains how to add database migrations to the Scopes project using SQLDelight. The migration system automatically updates database schemas when the application version changes. + +## Architecture + +The migration system consists of three main components: + +1. **DatabaseMigrationManager** (`platform/infrastructure`) - Handles migration execution +2. **ApplicationVersion** (`platform/infrastructure`) - Tracks schema versions +3. **Migration Files** (`.sqm` files) - Contains SQL migration statements + +## Adding a New Migration + +### Step 1: Create Migration File + +Create a new migration file in the appropriate context's `migrations` directory. The file should be named `{version}.sqm` where `{version}` is the current schema version. + +``` +contexts/{context}/infrastructure/src/main/sqldelight/migrations/{version}.sqm +``` + +Example locations: +- `contexts/scope-management/infrastructure/src/main/sqldelight/migrations/1.sqm` +- `contexts/event-store/infrastructure/src/main/sqldelight/migrations/1.sqm` +- `contexts/device-synchronization/infrastructure/src/main/sqldelight/migrations/1.sqm` + +### Step 2: Write Migration SQL + +Add your SQL migration statements to the file: + +```sql +-- Migration from version 1 to version 2 +-- Description of what this migration does + +-- Add new column +ALTER TABLE scopes ADD COLUMN status TEXT DEFAULT 'active'; + +-- Create new index +CREATE INDEX IF NOT EXISTS idx_scopes_status ON scopes(status); + +-- Create new table +CREATE TABLE IF NOT EXISTS scope_tags ( + scope_id TEXT NOT NULL, + tag TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (scope_id, tag), + FOREIGN KEY (scope_id) REFERENCES scopes(id) ON DELETE CASCADE +); +``` + +### Step 3: Update Schema Version + +Edit `ApplicationVersion.kt` to increment the schema version: + +```kotlin +object SchemaVersions { + const val SCOPE_MANAGEMENT = 2L // Incremented from 1L + const val EVENT_STORE = 1L + const val DEVICE_SYNCHRONIZATION = 1L + const val USER_PREFERENCES = 1L +} +``` + +### Step 4: Add Version Mapping + +Add a new entry to `versionHistory`: + +```kotlin +val versionHistory = listOf( + VersionMapping( + appVersion = "0.1.0", + scopeManagementSchema = 1L, + eventStoreSchema = 1L, + deviceSyncSchema = 1L, + userPreferencesSchema = 1L + ), + // New entry for the migration + VersionMapping( + appVersion = "0.2.0", + scopeManagementSchema = 2L, // Updated schema version + eventStoreSchema = 1L, + deviceSyncSchema = 1L, + userPreferencesSchema = 1L + ) +) +``` + +### Step 5: Update Application Version + +Update the `CURRENT_VERSION` constant: + +```kotlin +const val CURRENT_VERSION = "0.2.0" // Updated from "0.1.0" +``` + +## Migration Guidelines + +### DO: +- ✅ Test migrations thoroughly with production-like data +- ✅ Keep migrations idempotent (safe to run multiple times) +- ✅ Use `IF NOT EXISTS` for creating tables/indexes +- ✅ Provide default values for new NOT NULL columns +- ✅ Write clear comments explaining the migration purpose +- ✅ Consider data migration needs (not just schema changes) + +### DON'T: +- ❌ Use `BEGIN/END TRANSACTION` in migration files (handled automatically) +- ❌ Drop columns without considering data loss +- ❌ Make breaking changes without a migration strategy +- ❌ Forget to update both schema version and app version +- ❌ Skip testing rollback scenarios + +## Custom Migration Logic + +For complex data migrations, use callbacks: + +```kotlin +// In SqlDelightDatabaseProvider.kt +val callbacks = mapOf( + 2L to DatabaseMigrationManager.MigrationCallback { driver -> + // Custom migration logic + driver.execute( + null, + "UPDATE scopes SET status = 'archived' WHERE updated_at < ?", + 1, + bindLong(0, thirtyDaysAgo) + ) + } +) + +migrationManager.migrate( + driver = driver, + schema = ScopeManagementDatabase.Schema, + targetVersion = ApplicationVersion.SchemaVersions.SCOPE_MANAGEMENT, + callbacks = callbacks +) +``` + +## Testing Migrations + +### Unit Test + +Create a test for your migration: + +```kotlin +class MigrationTest : DescribeSpec({ + describe("Migration from v1 to v2") { + it("should add status column") { + val driver = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) + + // Create v1 schema + driver.execute(null, "CREATE TABLE scopes (id TEXT PRIMARY KEY)", 0) + driver.execute(null, "PRAGMA user_version = 1", 0) + + // Run migration + val manager = DatabaseMigrationManager.createDefault() + manager.migrate(driver, TestSchema, 2L) + + // Verify new column exists + val hasColumn = driver.executeQuery( + null, + "PRAGMA table_info(scopes)", + mapper = { cursor -> + QueryResult.Value( + buildList { + while (cursor.next().value) { + add(cursor.getString(1) ?: "") + } + }.contains("status") + ) + }, + 0 + ).value + + hasColumn shouldBe true + } + } +}) +``` + +### Manual Testing + +1. Copy a production database to test environment +2. Run the application with new version +3. Verify migration succeeds +4. Check data integrity +5. Test rollback if needed + +## Rollback Strategy + +While SQLDelight doesn't support automatic rollback, you can: + +1. **Backup before migration**: Always backup production databases +2. **Write reverse migrations**: Create SQL scripts to undo changes +3. **Use transactions**: Migrations run in transactions and rollback on failure +4. **Test thoroughly**: Prevent the need for rollbacks + +## Common Migration Scenarios + +### Adding a Column + +```sql +ALTER TABLE scopes ADD COLUMN priority INTEGER DEFAULT 0; +``` + +### Adding an Index + +```sql +CREATE INDEX IF NOT EXISTS idx_scopes_priority ON scopes(priority); +``` + +### Renaming a Column + +```sql +-- SQLite doesn't support RENAME COLUMN directly +-- Create new column, copy data, drop old column +ALTER TABLE scopes ADD COLUMN new_name TEXT; +UPDATE scopes SET new_name = old_name; +-- In next migration: ALTER TABLE scopes DROP COLUMN old_name; +``` + +### Adding a Foreign Key + +```sql +-- SQLite requires recreating the table for new foreign keys +-- Better to include in initial schema design +``` + +### Data Migration + +```sql +-- Update existing data +UPDATE scopes SET status = 'active' WHERE status IS NULL; +``` + +## Troubleshooting + +### Migration Fails + +1. Check SQL syntax in migration file +2. Verify schema version consistency +3. Check for constraint violations +4. Review migration logs + +### Version Mismatch + +If database version > application version: +- Database is from newer application version +- May need to upgrade application +- Check version history + +### Performance Issues + +- Add indexes after data migration +- Use batch operations for large updates +- Consider running migrations during maintenance windows + +## Best Practices + +1. **Version Control**: Always commit migration files with schema changes +2. **Documentation**: Document migration purpose and impact +3. **Backward Compatibility**: Consider older app versions during migration +4. **Testing**: Test with production-like data volumes +5. **Monitoring**: Log migration progress and timing +6. **Atomic Changes**: Keep migrations small and focused + +## Related Documentation + +- [SQLDelight Documentation](https://sqldelight.github.io/sqldelight/2.0.2/multiplatform_sqlite/migrations/) +- [Clean Architecture Guide](../explanation/clean-architecture.md) +- [Testing Guide](./testing.md) \ No newline at end of file diff --git a/docs/issues/enable-foreign-keys-in-tests.md b/docs/issues/enable-foreign-keys-in-tests.md new file mode 100644 index 000000000..f765bf4fe --- /dev/null +++ b/docs/issues/enable-foreign-keys-in-tests.md @@ -0,0 +1,40 @@ +# Enable Foreign Key Constraints in Test Databases + +## Issue + +The AI review correctly identified that foreign key constraints are not enabled for in-memory test databases, which could lead to test-production parity issues. While production databases enable foreign keys via `ManagedSqlDriver.createWithDefaults()`, in-memory test databases created by `createInMemoryDatabase()` do not. + +## Attempted Fix + +Adding `PRAGMA foreign_keys = ON` to in-memory database creation causes 23 test failures in `SqlDelightScopeAliasRepositoryTest` and 2 in `SqlDelightScopeRepositoryTest`. These tests are inserting alias records without corresponding scope records, violating foreign key constraints. + +## Required Changes + +1. **Enable foreign keys in all in-memory databases**: + - Add `driver.execute(null, "PRAGMA foreign_keys = ON", 0)` after driver creation + - Apply to all `createInMemoryDatabase()` methods across bounded contexts + +2. **Fix failing tests**: + - Update tests to properly create parent scope records before aliases + - Ensure test data respects referential integrity + - Consider using test fixtures or builders for proper test data setup + +## Benefits + +- Test-production parity: Tests will catch foreign key violations +- Data integrity: Prevents orphaned records in production +- Better test quality: Forces tests to use realistic data relationships + +## Implementation Notes + +The foreign key pragma must be set: +- After creating the driver +- Before creating the schema +- For each new connection (SQLite default is OFF) + +## Related Files + +- `contexts/scope-management/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/scopemanagement/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt` +- `contexts/event-store/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/eventstore/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt` +- `contexts/device-synchronization/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/devicesync/infrastructure/sqldelight/SqlDelightDatabaseProvider.kt` +- Test files that need updating for proper foreign key compliance \ No newline at end of file diff --git a/platform/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/platform/infrastructure/database/DatabaseMigrationManager.kt b/platform/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/platform/infrastructure/database/DatabaseMigrationManager.kt new file mode 100644 index 000000000..a90a7357a --- /dev/null +++ b/platform/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/platform/infrastructure/database/DatabaseMigrationManager.kt @@ -0,0 +1,218 @@ +package io.github.kamiazya.scopes.platform.infrastructure.database + +import app.cash.sqldelight.db.QueryResult +import app.cash.sqldelight.db.SqlDriver +import app.cash.sqldelight.db.SqlSchema +import io.github.kamiazya.scopes.platform.observability.logging.Logger +import io.github.kamiazya.scopes.platform.observability.logging.ConsoleLogger + +/** + * Manages database migrations for SQLDelight databases. + * + * This manager provides: + * - Automatic migration execution based on version differences + * - Safe migration with transaction support + * - Logging of migration progress + * - Version tracking in user_version PRAGMA + * - Thread-safe migration execution + */ +class DatabaseMigrationManager( + private val logger: Logger = ConsoleLogger() +) { + /** + * Executes database migration if needed. + * This method is synchronized to prevent concurrent migrations. + * + * @param driver The SQL driver to execute migrations on + * @param schema The SQLDelight schema containing migration logic + * @param targetVersion The target schema version to migrate to + * @param callbacks Optional callbacks for custom migration logic at specific versions + * @throws IllegalStateException if migration fails or database is newer than application + */ + @Synchronized + fun migrate( + driver: SqlDriver, + schema: SqlSchema<*>, + targetVersion: Long, + callbacks: Map = emptyMap() + ) { + // Use database-level locking to prevent concurrent migrations across processes + var transactionActive = false + try { + driver.execute(null, "BEGIN IMMEDIATE", 0) + transactionActive = true + performMigrationInternal(driver, schema, targetVersion, callbacks) + driver.execute(null, "COMMIT", 0) + transactionActive = false + } catch (e: Exception) { + if (transactionActive) { + try { + driver.execute(null, "ROLLBACK", 0) + } catch (rollbackException: Exception) { + // Log rollback failure but preserve original exception + logger.error("Failed to rollback migration transaction", throwable = rollbackException) + } + } + error("Migration failed: ${e.message}") + } + } + + /** + * Internal migration logic, called within database transaction lock. + */ + private fun performMigrationInternal( + driver: SqlDriver, + schema: SqlSchema<*>, + targetVersion: Long, + callbacks: Map + ) { + val currentVersion = getCurrentVersion(driver) + + when { + currentVersion == 0L && !isDatabaseEmpty(driver) -> { + // Database exists but no version set - likely pre-migration database + logger.warn("Database exists without version. Setting version to 1 and attempting migration.") + setVersion(driver, 1L) + if (targetVersion > 1L) { + executeSchemaUpdate(driver, schema, 1L, targetVersion, callbacks) + } + } + currentVersion == 0L -> { + // Fresh database + logger.info("Creating new database schema (version $targetVersion)") + try { + schema.create(driver) + setVersion(driver, targetVersion) + } catch (e: Exception) { + error("Failed to create database schema: ${e.message}") + } + } + currentVersion < targetVersion -> { + logger.info("Migrating database from version $currentVersion to $targetVersion") + executeSchemaUpdate(driver, schema, currentVersion, targetVersion, callbacks) + } + currentVersion > targetVersion -> { + // Fail fast when database is newer than application + val message = "Database version ($currentVersion) is newer than application version ($targetVersion). " + + "Please update the application to a newer version." + logger.error(message) + error(message) + } + else -> { + logger.debug("Database is up to date (version $currentVersion)") + } + } + } + + /** + * Checks if the database is empty (no tables). + */ + private fun isDatabaseEmpty(driver: SqlDriver): Boolean { + return driver.executeQuery( + identifier = null, + sql = "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'", + mapper = { cursor -> + QueryResult.Value( + if (cursor.next().value) { + cursor.getLong(0) == 0L + } else { + true + } + ) + }, + parameters = 0 + ).value + } + + /** + * Gets the current database schema version. + * SQLite stores this in the user_version PRAGMA. + */ + private fun getCurrentVersion(driver: SqlDriver): Long { + return driver.executeQuery( + identifier = null, + sql = "PRAGMA user_version", + mapper = { cursor -> + QueryResult.Value( + if (cursor.next().value) { + cursor.getLong(0) ?: 0L + } else { + 0L + } + ) + }, + parameters = 0 + ).value + } + + /** + * Sets the database schema version. + */ + private fun setVersion(driver: SqlDriver, version: Long) { + driver.execute( + identifier = null, + sql = "PRAGMA user_version = $version", + parameters = 0 + ) + } + + /** + * Performs the actual schema update from one version to another. + */ + private fun executeSchemaUpdate( + driver: SqlDriver, + schema: SqlSchema<*>, + currentVersion: Long, + targetVersion: Long, + callbacks: Map + ) { + try { + // Execute SQLDelight migrations + schema.migrate(driver, currentVersion, targetVersion) + + // Note: The callbacks parameter with afterVersion lambda is not part of the standard migrate API + // If custom callbacks are needed at specific versions, they should be handled separately + for (version in (currentVersion + 1)..targetVersion) { + callbacks[version]?.let { callback -> + logger.debug("Executing custom migration callback for version $version") + callback.execute(driver) + } + } + + // Update version + setVersion(driver, targetVersion) + + logger.info("Migration completed successfully") + } catch (e: Exception) { + logger.error("Migration failed", throwable = e) + error("Failed to migrate database from $currentVersion to $targetVersion: ${e.message}") + } + } + + /** + * Callback interface for custom migration logic. + */ + fun interface MigrationCallback { + /** + * Execute custom migration logic. + */ + fun execute(driver: SqlDriver) + } + + companion object { + /** + * Creates a migration manager with default configuration. + */ + fun createDefault(): DatabaseMigrationManager { + return DatabaseMigrationManager() + } + } +} + +/** + * Exception thrown when database migration fails. + */ +class DatabaseMigrationException( + message: String, + cause: Throwable? = null +) : RuntimeException(message, cause) \ No newline at end of file diff --git a/platform/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/platform/infrastructure/version/ApplicationVersion.kt b/platform/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/platform/infrastructure/version/ApplicationVersion.kt new file mode 100644 index 000000000..3f209bb3f --- /dev/null +++ b/platform/infrastructure/src/main/kotlin/io/github/kamiazya/scopes/platform/infrastructure/version/ApplicationVersion.kt @@ -0,0 +1,123 @@ +package io.github.kamiazya.scopes.platform.infrastructure.version + +/** + * Application version information and database schema versioning. + * + * This singleton provides: + * - Application semantic version + * - Database schema version mapping + * - Version comparison utilities + */ +object ApplicationVersion { + /** + * Current application version (semantic versioning). + * This should be updated with each release. + */ + const val CURRENT_VERSION = "0.1.0" + + /** + * Database schema versions for each bounded context. + * These versions are incremented when database schema changes. + */ + object SchemaVersions { + /** + * Scope Management context database schema version. + */ + const val SCOPE_MANAGEMENT = 1L + + /** + * Event Store context database schema version. + */ + const val EVENT_STORE = 1L + + /** + * Device Synchronization context database schema version. + */ + const val DEVICE_SYNCHRONIZATION = 1L + + /** + * User Preferences context database schema version. + * (When implemented with database storage) + */ + const val USER_PREFERENCES = 1L + } + + /** + * Maps application version to schema versions. + * This helps track which schema versions are compatible with which app versions. + */ + data class VersionMapping( + val appVersion: String, + val scopeManagementSchema: Long, + val eventStoreSchema: Long, + val deviceSyncSchema: Long, + val userPreferencesSchema: Long + ) + + /** + * Historical version mappings for reference. + * Add new entries when releasing versions with schema changes. + */ + val versionHistory = listOf( + VersionMapping( + appVersion = "0.1.0", + scopeManagementSchema = 1L, + eventStoreSchema = 1L, + deviceSyncSchema = 1L, + userPreferencesSchema = 1L + ) + // Add future versions here as they are released + // Example: + // VersionMapping( + // appVersion = "0.2.0", + // scopeManagementSchema = 2L, // Schema updated + // eventStoreSchema = 1L, // No change + // deviceSyncSchema = 1L, // No change + // userPreferencesSchema = 1L // No change + // ) + ) + + /** + * Gets the current version mapping. + */ + fun getCurrentMapping(): VersionMapping { + return versionHistory.last() + } + + /** + * Parses semantic version string into comparable parts. + */ + fun parseVersion(version: String): Triple { + val parts = version.split(".") + return Triple( + parts.getOrNull(0)?.toIntOrNull() ?: 0, + parts.getOrNull(1)?.toIntOrNull() ?: 0, + parts.getOrNull(2)?.toIntOrNull() ?: 0 + ) + } + + /** + * Compares two semantic version strings. + * Returns: + * - negative if version1 < version2 + * - zero if version1 == version2 + * - positive if version1 > version2 + */ + fun compareVersions(version1: String, version2: String): Int { + val v1 = parseVersion(version1) + val v2 = parseVersion(version2) + + return when { + v1.first != v2.first -> v1.first.compareTo(v2.first) + v1.second != v2.second -> v1.second.compareTo(v2.second) + else -> v1.third.compareTo(v2.third) + } + } + + /** + * Checks if the application version is compatible with a database version. + */ + fun isCompatible(databaseSchemaVersion: Long, contextSchemaVersion: Long): Boolean { + return databaseSchemaVersion <= contextSchemaVersion + } +} \ No newline at end of file diff --git a/platform/infrastructure/src/test/kotlin/io/github/kamiazya/scopes/platform/infrastructure/database/DatabaseMigrationManagerTest.kt b/platform/infrastructure/src/test/kotlin/io/github/kamiazya/scopes/platform/infrastructure/database/DatabaseMigrationManagerTest.kt new file mode 100644 index 000000000..f7b44fdbc --- /dev/null +++ b/platform/infrastructure/src/test/kotlin/io/github/kamiazya/scopes/platform/infrastructure/database/DatabaseMigrationManagerTest.kt @@ -0,0 +1,326 @@ +package io.github.kamiazya.scopes.platform.infrastructure.database + +import app.cash.sqldelight.db.AfterVersion +import app.cash.sqldelight.db.QueryResult +import app.cash.sqldelight.db.SqlDriver +import app.cash.sqldelight.db.SqlSchema +import app.cash.sqldelight.driver.jdbc.sqlite.JdbcSqliteDriver +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.DescribeSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe + +class DatabaseMigrationManagerTest : DescribeSpec({ + describe("DatabaseMigrationManager") { + lateinit var migrationManager: DatabaseMigrationManager + lateinit var driver: SqlDriver + + beforeEach { + migrationManager = DatabaseMigrationManager.createDefault() + // Create in-memory database for testing + driver = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) + } + + afterEach { + driver.close() + } + + describe("migrate") { + it("should create new schema when database is fresh") { + // Given + val targetVersion = 1L + val createCalled = mutableListOf() + val schema = object : SqlSchema> { + override val version = targetVersion + + override fun create(driver: SqlDriver): QueryResult { + createCalled.add(driver) + return QueryResult.Value(Unit) + } + + override fun migrate( + driver: SqlDriver, + oldVersion: Long, + newVersion: Long, + vararg callbacks: AfterVersion + ): QueryResult { + return QueryResult.Value(Unit) + } + } + + // When + migrationManager.migrate(driver, schema, targetVersion) + + // Then + createCalled.size shouldBe 1 + createCalled.first() shouldBe driver + + // Check version was set + val version = driver.executeQuery( + null, + "PRAGMA user_version", + mapper = { cursor -> + QueryResult.Value(if (cursor.next().value) cursor.getLong(0) ?: 0L else 0L) + }, + 0 + ).value + version shouldBe targetVersion + } + + it("should migrate schema when current version is lower") { + // Given + val currentVersion = 1L + val targetVersion = 3L + val migrateCalled = mutableListOf>() + + val schema = object : SqlSchema> { + override val version = targetVersion + + override fun create(driver: SqlDriver): QueryResult { + return QueryResult.Value(Unit) + } + + override fun migrate( + driver: SqlDriver, + oldVersion: Long, + newVersion: Long, + vararg callbacks: AfterVersion + ): QueryResult { + migrateCalled.add(Triple(driver, oldVersion, newVersion)) + return QueryResult.Value(Unit) + } + } + + // Set initial version + driver.execute(null, "PRAGMA user_version = $currentVersion", 0) + + // When + migrationManager.migrate(driver, schema, targetVersion) + + // Then + migrateCalled.size shouldBe 1 + val (migrateDriver, oldVer, newVer) = migrateCalled.first() + migrateDriver shouldBe driver + oldVer shouldBe currentVersion + newVer shouldBe targetVersion + + // Check version was updated + val version = driver.executeQuery( + null, + "PRAGMA user_version", + mapper = { cursor -> + QueryResult.Value(if (cursor.next().value) cursor.getLong(0) ?: 0L else 0L) + }, + 0 + ).value + version shouldBe targetVersion + } + + it("should not migrate when database is up to date") { + // Given + val targetVersion = 2L + var createCalled = false + var migrateCalled = false + + val schema = object : SqlSchema> { + override val version = targetVersion + + override fun create(driver: SqlDriver): QueryResult { + createCalled = true + return QueryResult.Value(Unit) + } + + override fun migrate( + driver: SqlDriver, + oldVersion: Long, + newVersion: Long, + vararg callbacks: AfterVersion + ): QueryResult { + migrateCalled = true + return QueryResult.Value(Unit) + } + } + + // Set current version to target + driver.execute(null, "PRAGMA user_version = $targetVersion", 0) + + // When + migrationManager.migrate(driver, schema, targetVersion) + + // Then + createCalled shouldBe false + migrateCalled shouldBe false + } + + it("should execute custom callbacks during migration") { + // Given + val currentVersion = 1L + val targetVersion = 3L + var callbackExecuted = false + + val callbacks = mapOf( + 2L to DatabaseMigrationManager.MigrationCallback { _ -> + callbackExecuted = true + } + ) + + val schema = object : SqlSchema> { + override val version = targetVersion + + override fun create(driver: SqlDriver): QueryResult { + return QueryResult.Value(Unit) + } + + override fun migrate( + driver: SqlDriver, + oldVersion: Long, + newVersion: Long, + vararg callbacks: AfterVersion + ): QueryResult { + return QueryResult.Value(Unit) + } + } + + // Set initial version + driver.execute(null, "PRAGMA user_version = $currentVersion", 0) + + // When + migrationManager.migrate(driver, schema, targetVersion, callbacks) + + // Then + callbackExecuted shouldBe true + } + + it("should rollback on migration failure") { + // Given + val currentVersion = 1L + val targetVersion = 2L + + val schema = object : SqlSchema> { + override val version = targetVersion + + override fun create(driver: SqlDriver): QueryResult { + return QueryResult.Value(Unit) + } + + override fun migrate( + driver: SqlDriver, + oldVersion: Long, + newVersion: Long, + vararg callbacks: AfterVersion + ): QueryResult { + throw RuntimeException("Migration failed") + } + } + + // Set initial version + driver.execute(null, "PRAGMA user_version = $currentVersion", 0) + + // When & Then + val exception = shouldThrow { + migrationManager.migrate(driver, schema, targetVersion) + } + exception.message shouldBe "Migration failed: Failed to migrate database from 1 to 2: Migration failed" + + // Then - version should remain unchanged + val version = driver.executeQuery( + null, + "PRAGMA user_version", + mapper = { cursor -> + QueryResult.Value(if (cursor.next().value) cursor.getLong(0) ?: 0L else 0L) + }, + 0 + ).value + version shouldBe currentVersion + } + + it("should fail fast when database version is newer than target version") { + // Given + val currentVersion = 5L + val targetVersion = 3L + var createCalled = false + var migrateCalled = false + + val schema = object : SqlSchema> { + override val version = targetVersion + + override fun create(driver: SqlDriver): QueryResult { + createCalled = true + return QueryResult.Value(Unit) + } + + override fun migrate( + driver: SqlDriver, + oldVersion: Long, + newVersion: Long, + vararg callbacks: AfterVersion + ): QueryResult { + migrateCalled = true + return QueryResult.Value(Unit) + } + } + + // Set database version higher than target + driver.execute(null, "PRAGMA user_version = $currentVersion", 0) + + // When & Then + val exception = shouldThrow { + migrationManager.migrate(driver, schema, targetVersion) + } + + exception.message shouldBe "Migration failed: Database version ($currentVersion) is newer than application version ($targetVersion). Please update the application to a newer version." + + // Verify no schema operations were attempted + createCalled shouldBe false + migrateCalled shouldBe false + + // Version should remain unchanged + val version = driver.executeQuery( + null, + "PRAGMA user_version", + mapper = { cursor -> + QueryResult.Value(if (cursor.next().value) cursor.getLong(0) ?: 0L else 0L) + }, + 0 + ).value + version shouldBe currentVersion + } + } + + describe("getCurrentVersion") { + it("should return 0 for new database") { + // When + val version = driver.executeQuery( + null, + "PRAGMA user_version", + mapper = { cursor -> + QueryResult.Value(if (cursor.next().value) cursor.getLong(0) ?: 0L else 0L) + }, + 0 + ).value + + // Then + version shouldBe 0L + } + + it("should return correct version after setting") { + // Given + val expectedVersion = 5L + driver.execute(null, "PRAGMA user_version = $expectedVersion", 0) + + // When + val version = driver.executeQuery( + null, + "PRAGMA user_version", + mapper = { cursor -> + QueryResult.Value(if (cursor.next().value) cursor.getLong(0) ?: 0L else 0L) + }, + 0 + ).value + + // Then + version shouldBe expectedVersion + } + } + } +}) \ No newline at end of file diff --git a/platform/infrastructure/src/test/kotlin/io/github/kamiazya/scopes/platform/infrastructure/version/ApplicationVersionTest.kt b/platform/infrastructure/src/test/kotlin/io/github/kamiazya/scopes/platform/infrastructure/version/ApplicationVersionTest.kt new file mode 100644 index 000000000..c2def9673 --- /dev/null +++ b/platform/infrastructure/src/test/kotlin/io/github/kamiazya/scopes/platform/infrastructure/version/ApplicationVersionTest.kt @@ -0,0 +1,91 @@ +package io.github.kamiazya.scopes.platform.infrastructure.version + +import io.kotest.core.spec.style.DescribeSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe + +class ApplicationVersionTest : DescribeSpec({ + describe("ApplicationVersion") { + describe("version constants") { + it("should have a current version") { + ApplicationVersion.CURRENT_VERSION shouldNotBe null + ApplicationVersion.CURRENT_VERSION shouldBe "0.1.0" + } + + it("should have schema versions for each context") { + ApplicationVersion.SchemaVersions.SCOPE_MANAGEMENT shouldBe 1L + ApplicationVersion.SchemaVersions.EVENT_STORE shouldBe 1L + ApplicationVersion.SchemaVersions.DEVICE_SYNCHRONIZATION shouldBe 1L + ApplicationVersion.SchemaVersions.USER_PREFERENCES shouldBe 1L + } + } + + describe("parseVersion") { + it("should parse semantic version correctly") { + ApplicationVersion.parseVersion("1.2.3") shouldBe Triple(1, 2, 3) + ApplicationVersion.parseVersion("10.20.30") shouldBe Triple(10, 20, 30) + ApplicationVersion.parseVersion("0.1.0") shouldBe Triple(0, 1, 0) + } + + it("should handle incomplete versions") { + ApplicationVersion.parseVersion("1") shouldBe Triple(1, 0, 0) + ApplicationVersion.parseVersion("1.2") shouldBe Triple(1, 2, 0) + ApplicationVersion.parseVersion("") shouldBe Triple(0, 0, 0) + } + + it("should handle invalid versions") { + ApplicationVersion.parseVersion("abc") shouldBe Triple(0, 0, 0) + ApplicationVersion.parseVersion("1.x.3") shouldBe Triple(1, 0, 3) + ApplicationVersion.parseVersion("1.2.x") shouldBe Triple(1, 2, 0) + } + } + + describe("compareVersions") { + it("should compare versions correctly") { + // Equal + ApplicationVersion.compareVersions("1.2.3", "1.2.3") shouldBe 0 + + // Major version difference + ApplicationVersion.compareVersions("2.0.0", "1.0.0") shouldBe 1 + ApplicationVersion.compareVersions("1.0.0", "2.0.0") shouldBe -1 + + // Minor version difference + ApplicationVersion.compareVersions("1.3.0", "1.2.0") shouldBe 1 + ApplicationVersion.compareVersions("1.2.0", "1.3.0") shouldBe -1 + + // Patch version difference + ApplicationVersion.compareVersions("1.2.4", "1.2.3") shouldBe 1 + ApplicationVersion.compareVersions("1.2.3", "1.2.4") shouldBe -1 + } + } + + describe("getCurrentMapping") { + it("should return the latest version mapping") { + val mapping = ApplicationVersion.getCurrentMapping() + mapping.appVersion shouldBe "0.1.0" + mapping.scopeManagementSchema shouldBe 1L + mapping.eventStoreSchema shouldBe 1L + mapping.deviceSyncSchema shouldBe 1L + mapping.userPreferencesSchema shouldBe 1L + } + } + + describe("isCompatible") { + it("should check database compatibility") { + // Compatible cases + ApplicationVersion.isCompatible(1L, 1L) shouldBe true // Same version + ApplicationVersion.isCompatible(1L, 2L) shouldBe true // Database older than context + + // Incompatible case + ApplicationVersion.isCompatible(3L, 2L) shouldBe false // Database newer than context + } + } + + describe("versionHistory") { + it("should contain at least one version mapping") { + ApplicationVersion.versionHistory.size shouldBe 1 + ApplicationVersion.versionHistory.first().appVersion shouldBe "0.1.0" + } + } + } +}) \ No newline at end of file