From dc55b4f50f9394a136546d4da60ea62434ee715b Mon Sep 17 00:00:00 2001 From: Jurriaan Mous Date: Sun, 11 Feb 2024 22:59:29 +0100 Subject: [PATCH] Use one conversion context for all conversions. And fix an issue with keyDefinition comparison. --- .../kotlin/maryk/core/models/RootDataModel.kt | 2 +- .../kotlin/maryk/datastore/hbase/HbaseDataStore.kt | 5 ++++- .../hbase/model/checkModelIfMigrationIsNeeded.kt | 9 +++++---- .../kotlin/maryk/datastore/rocksdb/RocksDBDataStore.kt | 5 ++++- .../rocksdb/model/checkModelIfMigrationIsNeeded.kt | 10 +++++----- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/core/src/commonMain/kotlin/maryk/core/models/RootDataModel.kt b/core/src/commonMain/kotlin/maryk/core/models/RootDataModel.kt index 6fdb9e5c1..037a63159 100644 --- a/core/src/commonMain/kotlin/maryk/core/models/RootDataModel.kt +++ b/core/src/commonMain/kotlin/maryk/core/models/RootDataModel.kt @@ -117,7 +117,7 @@ open class RootDataModel internal constructor( migrationReasons += "Major version was increased: ${storedDataModel.Meta.version} -> ${this.Meta.version}" } - if (storedDataModel.Meta.keyDefinition !== this.Meta.keyDefinition) { + if (storedDataModel.Meta.keyDefinition != this.Meta.keyDefinition) { migrationReasons += "Key definition was not the same" } } else { diff --git a/store/hbase/src/commonMain/kotlin/maryk/datastore/hbase/HbaseDataStore.kt b/store/hbase/src/commonMain/kotlin/maryk/datastore/hbase/HbaseDataStore.kt index e0b6fc419..ba01fbe8e 100644 --- a/store/hbase/src/commonMain/kotlin/maryk/datastore/hbase/HbaseDataStore.kt +++ b/store/hbase/src/commonMain/kotlin/maryk/datastore/hbase/HbaseDataStore.kt @@ -15,6 +15,7 @@ import maryk.core.models.migration.MigrationStatus import maryk.core.models.migration.StoredRootDataModelDefinition import maryk.core.models.migration.VersionUpdateHandler import maryk.core.properties.definitions.index.IsIndexable +import maryk.core.query.DefinitionsConversionContext import maryk.core.query.requests.AddRequest import maryk.core.query.requests.ChangeRequest import maryk.core.query.requests.DeleteRequest @@ -97,10 +98,12 @@ class HbaseDataStore( ).await() } + val conversionContext = DefinitionsConversionContext() + for (dataModel in dataModelsById.values) { val tableName = getTableName(dataModel) val tableDescriptor = admin.getDescriptor(tableName) - when (val migrationStatus = checkModelIfMigrationIsNeeded(tableDescriptor, dataModel, onlyCheckModelVersion)) { + when (val migrationStatus = checkModelIfMigrationIsNeeded(tableDescriptor, dataModel, onlyCheckModelVersion, conversionContext)) { MigrationStatus.UpToDate, MigrationStatus.AlreadyProcessed -> Unit // Do nothing since no work is needed MigrationStatus.NewModel -> { scheduledVersionUpdateHandlers.add { diff --git a/store/hbase/src/commonMain/kotlin/maryk/datastore/hbase/model/checkModelIfMigrationIsNeeded.kt b/store/hbase/src/commonMain/kotlin/maryk/datastore/hbase/model/checkModelIfMigrationIsNeeded.kt index 6de7b2c31..f44070dcf 100644 --- a/store/hbase/src/commonMain/kotlin/maryk/datastore/hbase/model/checkModelIfMigrationIsNeeded.kt +++ b/store/hbase/src/commonMain/kotlin/maryk/datastore/hbase/model/checkModelIfMigrationIsNeeded.kt @@ -18,7 +18,8 @@ import java.util.concurrent.CompletableFuture suspend fun checkModelIfMigrationIsNeeded( tableDescriptorFuture: CompletableFuture, dataModel: IsRootDataModel, - onlyCheckVersion: Boolean + onlyCheckVersion: Boolean, + conversionContext: DefinitionsConversionContext, ): MigrationStatus { val tableDescriptor = try { tableDescriptorFuture.await() @@ -38,12 +39,11 @@ suspend fun checkModelIfMigrationIsNeeded( return when { dataModel.Meta.version != version || !onlyCheckVersion -> { - val context = DefinitionsConversionContext() // Read currently stored dependent model tableDescriptor.getValue(TableMetaColumns.Dependents.byteArray)?.let { dependentModelBytes -> var readIndex = 0 - Definitions.Serializer.readProtoBuf(dependentModelBytes.size, { dependentModelBytes[readIndex++] }, context).toDataObject() + Definitions.Serializer.readProtoBuf(dependentModelBytes.size, { dependentModelBytes[readIndex++] }, conversionContext).toDataObject() } // Read currently stored model @@ -51,7 +51,8 @@ suspend fun checkModelIfMigrationIsNeeded( ?: throw StorageException("Model is unexpectedly missing in metadata for ${dataModel.Meta.name}") var readIndex = 0 - val storedDataModel = RootDataModel.Model.Serializer.readProtoBuf(modelBytes.size, { modelBytes[readIndex++] }, context).toDataObject() + val storedDataModel = RootDataModel.Model.Serializer.readProtoBuf(modelBytes.size, { modelBytes[readIndex++] }, conversionContext).toDataObject() + conversionContext.dataModels[dataModel.Meta.name] = { storedDataModel } // Check by comparing the data models for if migration is needed return dataModel.isMigrationNeeded(storedDataModel) diff --git a/store/rocksdb/src/commonMain/kotlin/maryk/datastore/rocksdb/RocksDBDataStore.kt b/store/rocksdb/src/commonMain/kotlin/maryk/datastore/rocksdb/RocksDBDataStore.kt index 9cca65186..5d2c31e3a 100644 --- a/store/rocksdb/src/commonMain/kotlin/maryk/datastore/rocksdb/RocksDBDataStore.kt +++ b/store/rocksdb/src/commonMain/kotlin/maryk/datastore/rocksdb/RocksDBDataStore.kt @@ -22,6 +22,7 @@ import maryk.core.models.migration.MigrationStatus.UpToDate import maryk.core.models.migration.StoredRootDataModelDefinition import maryk.core.models.migration.VersionUpdateHandler import maryk.core.properties.definitions.index.IsIndexable +import maryk.core.query.DefinitionsConversionContext import maryk.core.query.requests.AddRequest import maryk.core.query.requests.ChangeRequest import maryk.core.query.requests.DeleteRequest @@ -160,10 +161,12 @@ class RocksDBDataStore( runBlocking { launch(Dispatchers.IO) { + val conversionContext = DefinitionsConversionContext() + for ((index, dataModel) in dataModelsById) { columnFamilyHandlesByDataModelIndex[index]?.let { tableColumnFamilies -> tableColumnFamilies.model.let { modelColumnFamily -> - when (val migrationStatus = checkModelIfMigrationIsNeeded(db, modelColumnFamily, dataModel, onlyCheckModelVersion)) { + when (val migrationStatus = checkModelIfMigrationIsNeeded(db, modelColumnFamily, dataModel, onlyCheckModelVersion, conversionContext)) { UpToDate, MigrationStatus.AlreadyProcessed -> Unit // Do nothing since no work is needed NewModel -> { scheduledVersionUpdateHandlers.add { diff --git a/store/rocksdb/src/commonMain/kotlin/maryk/datastore/rocksdb/model/checkModelIfMigrationIsNeeded.kt b/store/rocksdb/src/commonMain/kotlin/maryk/datastore/rocksdb/model/checkModelIfMigrationIsNeeded.kt index 4c8237e92..4fdf2c565 100644 --- a/store/rocksdb/src/commonMain/kotlin/maryk/datastore/rocksdb/model/checkModelIfMigrationIsNeeded.kt +++ b/store/rocksdb/src/commonMain/kotlin/maryk/datastore/rocksdb/model/checkModelIfMigrationIsNeeded.kt @@ -16,7 +16,8 @@ fun checkModelIfMigrationIsNeeded( rocksDB: RocksDB, modelColumnFamily: ColumnFamilyHandle, dataModel: IsRootDataModel, - onlyCheckVersion: Boolean + onlyCheckVersion: Boolean, + conversionContext: DefinitionsConversionContext, ): MigrationStatus { val name = rocksDB.get(modelColumnFamily, modelNameKey)?.decodeToString() val version = rocksDB.get(modelColumnFamily, modelVersionKey)?.let { @@ -30,12 +31,10 @@ fun checkModelIfMigrationIsNeeded( return when { dataModel.Meta.version != version || !onlyCheckVersion -> { - val context = DefinitionsConversionContext() - // Read currently stored dependent model rocksDB.get(modelColumnFamily, modelDependentsDefinitionKey)?.let { dependentModelBytes -> var readIndex = 0 - Definitions.Serializer.readProtoBuf(dependentModelBytes.size, { dependentModelBytes[readIndex++] }, context).toDataObject() + Definitions.Serializer.readProtoBuf(dependentModelBytes.size, { dependentModelBytes[readIndex++] }, conversionContext).toDataObject() } // Read currently stored model @@ -43,7 +42,8 @@ fun checkModelIfMigrationIsNeeded( ?: throw StorageException("Model is unexpectedly missing in metadata for ${dataModel.Meta.name}") var readIndex = 0 - val storedDataModel = RootDataModel.Model.Serializer.readProtoBuf(modelBytes.size, { modelBytes[readIndex++] }, context).toDataObject() + val storedDataModel = RootDataModel.Model.Serializer.readProtoBuf(modelBytes.size, { modelBytes[readIndex++] }, conversionContext).toDataObject() + conversionContext.dataModels[dataModel.Meta.name] = { storedDataModel } // Check by comparing the data models for if migration is needed return dataModel.isMigrationNeeded(storedDataModel)