Improve model definition storage so it also stores dependent models/enums. Add tests for those types of migration.

Author: jurmous

core/src/commonMain/kotlin/maryk/core/definitions/Definitions.kt

@@ -41,7 +41,7 @@ data class Definitions(
 
     constructor(vararg definition: MarykPrimitive) : this(definition.toList())
 
-    internal companion object : SingleValueDataModel<List<TypedValue<PrimitiveType, MarykPrimitive>>, List<MarykPrimitive>, Definitions, Companion, ContainsDefinitionsContext>(
+    companion object : SingleValueDataModel<List<TypedValue<PrimitiveType, MarykPrimitive>>, List<MarykPrimitive>, Definitions, Companion, ContainsDefinitionsContext>(
         { Companion.definitions }
     ) {
         val definitions by list(

core/src/commonMain/kotlin/maryk/core/properties/definitions/IsPropertyDefinition.kt

@@ -47,7 +47,7 @@ interface IsPropertyDefinition<T : Any> {
         addIncompatibilityReason: ((String) -> Unit)? = null
     ): Boolean {
         var compatible = true
-        if (this::class != definition::class) {
+        if (!this::class.isInstance(definition)) {
             addIncompatibilityReason?.invoke("Definitions are not of same types: ${this::class.simpleName} vs ${definition::class.simpleName}")
             compatible = false
         }

store/rocksdb/src/commonMain/kotlin/maryk/datastore/rocksdb/RocksDBDataStore.kt

@@ -163,10 +163,10 @@ class RocksDBDataStore(
                             }
                             is NeedsMigration -> {
                                 val succeeded = migrationHandler?.invoke(this, migrationStatus.storedDataModel as StoredRootDataModelDefinition, dataModel)
-                                    ?: throw MigrationException("Migration needed: No migration handler present")
+                                    ?: throw MigrationException("Migration needed: No migration handler present. \n$migrationStatus")
 
                                 if (!succeeded) {
-                                    throw MigrationException("Migration could not be handled for ${dataModel.Meta.name} & ${(migrationStatus.storedDataModel as? StoredRootDataModelDefinition)?.Meta?.version}")
+                                    throw MigrationException("Migration could not be handled for ${dataModel.Meta.name} & ${(migrationStatus.storedDataModel as? StoredRootDataModelDefinition)?.Meta?.version}\n$migrationStatus")
                                 }
 
                                 migrationStatus.indicesToIndex?.let {

store/rocksdb/src/commonMain/kotlin/maryk/datastore/rocksdb/model/checkModelIfMigrationIsNeeded.kt

@@ -1,5 +1,6 @@
 package maryk.datastore.rocksdb.model
 
+import maryk.core.definitions.Definitions
 import maryk.core.exceptions.StorageException
 import maryk.core.models.IsRootDataModel
 import maryk.core.models.RootDataModel
@@ -29,12 +30,19 @@ fun checkModelIfMigrationIsNeeded(
 
     return when {
         dataModel.Meta.version != version || !onlyCheckVersion -> {
+            val context = DefinitionsConversionContext()
+
+            // Read currently stored dependent model
+            rocksDB.get(modelColumnFamily, modelDependentsDefinitionKey)?.let { dependentModelBytes ->
+                var readIndex = 0
+                Definitions.Serializer.readProtoBuf(dependentModelBytes.size, { dependentModelBytes[readIndex++] }, context).toDataObject()
+            }
+
             // Read currently stored model
             val modelBytes = rocksDB.get(modelColumnFamily, modelDefinitionKey)
                 ?: throw StorageException("Model is unexpectedly missing in metadata for ${dataModel.Meta.name}")
 
             var readIndex = 0
-            val context = DefinitionsConversionContext()
             val storedDataModel = RootDataModel.Model.Serializer.readProtoBuf(modelBytes.size, { modelBytes[readIndex++] }, context).toDataObject()
 
             // Check by comparing the data models for if migration is needed

store/rocksdb/src/commonMain/kotlin/maryk/datastore/rocksdb/model/modelKeys.kt

@@ -3,3 +3,4 @@ package maryk.datastore.rocksdb.model
 internal val modelNameKey = byteArrayOf(0)
 internal val modelVersionKey = byteArrayOf(1)
 internal val modelDefinitionKey = byteArrayOf(2)
+internal val modelDependentsDefinitionKey = byteArrayOf(3)

store/rocksdb/src/commonMain/kotlin/maryk/datastore/rocksdb/model/storeModelDefinition.kt

@@ -1,5 +1,7 @@
 package maryk.datastore.rocksdb.model
 
+import maryk.core.definitions.Definitions
+import maryk.core.definitions.MarykPrimitive
 import maryk.core.models.IsRootDataModel
 import maryk.core.models.RootDataModel
 import maryk.core.protobuf.WriteCache
@@ -16,11 +18,28 @@ fun storeModelDefinition(
     rocksDB.put(modelColumnFamily, modelVersionKey, dataModel.Meta.version.toByteArray())
 
     val context = DefinitionsConversionContext()
-    val cache = WriteCache()
-    val modelByteSize = RootDataModel.Model.Serializer.calculateObjectProtoBufLength(dataModel as RootDataModel<*>, cache, context)
+
+    val modelCache = WriteCache()
+    val modelByteSize = RootDataModel.Model.Serializer.calculateObjectProtoBufLength(dataModel as RootDataModel<*>, modelCache, context)
+
+    if (context.dataModels.isNotEmpty()) {
+        val dependentCache = WriteCache()
+        val dependents = Definitions(buildList {
+            context.dataModels.values.forEach { add(it(Unit) as MarykPrimitive) }
+            context.enums.values.forEach { add(it as MarykPrimitive) }
+            context.typeEnums.values.forEach { add(it as MarykPrimitive) }
+        })
+        val depModelByteSize = Definitions.Serializer.calculateObjectProtoBufLength(dependents, dependentCache, context)
+        val dependentBytes = ByteArray(depModelByteSize)
+        var depWriteIndex = 0
+        Definitions.Serializer.writeObjectProtoBuf(dependents, dependentCache, { dependentBytes[depWriteIndex++] = it }, context)
+
+        rocksDB.put(modelColumnFamily, modelDependentsDefinitionKey, dependentBytes)
+    }
+
     val bytes = ByteArray(modelByteSize)
     var writeIndex = 0
-    RootDataModel.Model.Serializer.writeObjectProtoBuf(dataModel, cache, { bytes[writeIndex++] = it }, context)
+    RootDataModel.Model.Serializer.writeObjectProtoBuf(dataModel, modelCache, { bytes[writeIndex++] = it }, context)
 
     rocksDB.put(modelColumnFamily, modelDefinitionKey, bytes)
 }

store/rocksdb/src/commonTest/kotlin/maryk/datastore/rocksdb/RocksDBDataStoreMigrationTest.kt

@@ -16,6 +16,7 @@ import maryk.test.models.ModelV1
 import maryk.test.models.ModelV1_1
 import maryk.test.models.ModelV2
 import maryk.test.models.ModelV2ExtraIndex
+import maryk.test.models.ModelWithDependents
 import maryk.test.runSuspendingTest
 import kotlin.test.Test
 import kotlin.test.assertEquals
@@ -81,6 +82,30 @@ class RocksDBDataStoreMigrationTest {
         Unit
     }
 
+    @Test
+    fun testMigrationWithDependents() = runSuspendingTest {
+        val path = "$basePath/migrationWithDeps"
+        var dataStore = RocksDBDataStore(
+            keepAllVersions = true,
+            relativePath = path,
+            dataModelsById = mapOf(
+                1u to ModelWithDependents,
+            )
+        )
+
+        dataStore.close()
+
+        dataStore = RocksDBDataStore(
+            keepAllVersions = true,
+            relativePath = path,
+            dataModelsById = mapOf(
+                1u to ModelWithDependents,
+            )
+        )
+
+        dataStore.close()
+    }
+
     @Test
     fun testMigrationWithIndex() = runSuspendingTest {
         val path = "$basePath/migration2"

testmodels/src/commonMain/kotlin/maryk/test/models/MigrationDataModels.kt

@@ -2,7 +2,10 @@
 
 package maryk.test.models
 
+import maryk.core.models.DataModel
 import maryk.core.models.RootDataModel
+import maryk.core.properties.definitions.embed
+import maryk.core.properties.definitions.enum
 import maryk.core.properties.definitions.number
 import maryk.core.properties.definitions.string
 import maryk.core.properties.types.Version
@@ -86,3 +89,14 @@ object ModelV2ReservedNamesAndIndices : RootDataModel<ModelV2ReservedNamesAndInd
 ) {
     val newNumber by number(index = 2u, type = SInt32, required = false)
 }
+
+object DependentModel : DataModel<DependentModel>() {
+    val value by string(index = 1u, default = "haha", regEx = "ha.*")
+}
+
+object ModelWithDependents : RootDataModel<ModelV2ReservedNamesAndIndices>(
+    version = Version(1, 0),
+) {
+    val dep by embed(index = 1u, dataModel = { DependentModel })
+    val enum by enum(index = 2u, enum = Option)
+}