Updated datastore root mutations to maintain change records for historical tracking

Author: dimitribouniol

README.md

@@ -45,13 +45,12 @@ targets: [
 `CodableDatastore` is a collection of types that make it easy to interface with large data stores of independent types without loading the entire data store in memory.
 
 > **Warning**
-> DO NOT USE THIS IN PRODOCUTION PROJECTS. As this project is currently still in its alpha phase, I cannot stress how important it is to not ship anything that relies on this code, or you will experience data loss. There is a chance the underlying model may continue to change day to day, or I will not be able to ever finish it.
+> DO NOT USE THIS IN PRODUCTION PROJECTS. As this project is currently still in its alpha phase, I cannot stress how important it is to not ship anything that relies on this code, or you will experience data loss. There is a chance the underlying model may continue to change day to day, or I will not be able to ever finish it.
 > Until then, please enjoy the code as a spectator or play around with it in toy projects to submit feedback!
 
 ### Road to 0.1 Betas
 
 As this project matures towards its first beta, a number of features still need to be fleshed out:
-- Fleshing out historical edit metadata
 - Migrating entries
 
 The above list will be kept up to date during development and will likely see additions during that process.

Sources/CodableDatastore/Persistence/Disk Persistence/Datastore/DatastoreRoot.swift

@@ -155,6 +155,13 @@ extension DiskPersistence.Datastore.RootObject {
 
         var createdIndexes: Set<DiskPersistence.Datastore.Index> = []
 
+        if isPersisted {
+            manifest.removedIndexes = []
+            manifest.removedIndexManifests = []
+            manifest.addedIndexes = []
+            manifest.addedIndexManifests = []
+        }
+        
         for (_, indexDescriptor) in descriptor.directIndexes {
             let key = indexDescriptor.key
             let indexType = indexDescriptor.indexType
@@ -177,6 +184,9 @@ extension DiskPersistence.Datastore.RootObject {
                     )
                 )
                 createdIndexes.insert(index)
+                manifest.addedIndexes.insert(.direct(index: indexInfo.id))
+                manifest.addedIndexManifests.insert(.direct(index: indexInfo.id, manifest: indexInfo.root))
+                manifest.directIndexManifests.append(indexInfo)
             }
 
             manifest.descriptor.directIndexes[key] = DatastoreDescriptor.IndexDescriptor(
@@ -208,6 +218,9 @@ extension DiskPersistence.Datastore.RootObject {
                     )
                 )
                 createdIndexes.insert(index)
+                manifest.addedIndexes.insert(.secondary(index: indexInfo.id))
+                manifest.addedIndexManifests.insert(.secondary(index: indexInfo.id, manifest: indexInfo.root))
+                manifest.secondaryIndexManifests.append(indexInfo)
             }
 
             manifest.descriptor.secondaryIndexes[key] = DatastoreDescriptor.IndexDescriptor(
@@ -218,10 +231,12 @@ extension DiskPersistence.Datastore.RootObject {
         }
 
         if originalManifest.descriptor != manifest.descriptor {
-            manifest.id = DatastoreRootIdentifier()
-            manifest.modificationDate = Date()
+            let modificationDate = Date()
+            manifest.id = DatastoreRootIdentifier(date: modificationDate)
+            manifest.modificationDate = modificationDate
+            return (manifest: manifest, createdIndexes: createdIndexes)
         }
-        return (manifest: manifest, createdIndexes: createdIndexes)
+        return (manifest: originalManifest, createdIndexes: createdIndexes)
     }
 
     func manifest(
@@ -230,30 +245,62 @@ extension DiskPersistence.Datastore.RootObject {
         let manifest = try await manifest
         var updatedManifest = manifest
 
+        var removedIndex: DatastoreRootManifest.IndexManifestID?
+        var addedIndex: DatastoreRootManifest.IndexManifestID
+        
         switch index {
         case .primary(let manifestID):
+            removedIndex = .primary(manifest: updatedManifest.primaryIndexManifest)
+            addedIndex = .primary(manifest: manifestID)
             updatedManifest.primaryIndexManifest = manifestID
         case .direct(let indexID, let manifestID):
+            var oldRoot: DatastoreIndexManifestIdentifier?
             updatedManifest.directIndexManifests = manifest.directIndexManifests.map { indexInfo in
                 var indexInfo = indexInfo
                 if indexInfo.id == indexID {
+                    oldRoot = indexInfo.root
                     indexInfo.root = manifestID
                 }
                 return indexInfo
             }
+            
+            removedIndex = oldRoot.map { .direct(index: indexID, manifest: $0) }
+            addedIndex = .direct(index: indexID, manifest: manifestID)
         case .secondary(let indexID, let manifestID):
+            var oldRoot: DatastoreIndexManifestIdentifier?
             updatedManifest.secondaryIndexManifests = updatedManifest.secondaryIndexManifests.map { indexInfo in
                 var indexInfo = indexInfo
                 if indexInfo.id == indexID {
+                    oldRoot = indexInfo.root
                     indexInfo.root = manifestID
                 }
                 return indexInfo
             }
+            
+            removedIndex = oldRoot.map { .secondary(index: indexID, manifest: $0) }
+            addedIndex = .secondary(index: indexID, manifest: manifestID)
         }
 
         if manifest != updatedManifest {
-            updatedManifest.id = DatastoreRootIdentifier()
-            updatedManifest.modificationDate = Date()
+            let modificationDate = Date()
+            updatedManifest.id = DatastoreRootIdentifier(date: modificationDate)
+            updatedManifest.modificationDate = modificationDate
+            
+            if isPersisted {
+                updatedManifest.removedIndexes = []
+                updatedManifest.removedIndexManifests = []
+                updatedManifest.addedIndexes = []
+                updatedManifest.addedIndexManifests = []
+            }
+            
+            if let removedIndex {
+                if updatedManifest.addedIndexManifests.contains(removedIndex) {
+                    updatedManifest.addedIndexManifests.remove(removedIndex)
+                } else {
+                    updatedManifest.removedIndexManifests.insert(removedIndex)
+                }
+            }
+            updatedManifest.addedIndexManifests.insert(addedIndex)
         }
         return updatedManifest
     }

Sources/CodableDatastore/Persistence/Disk Persistence/Datastore/DatastoreRootManifest.swift

@@ -36,6 +36,18 @@ struct DatastoreRootManifest: Codable, Equatable, Identifiable {
 
     /// A pointer to the secondary indexes' root objects.
     var secondaryIndexManifests: [IndexInfo] = []
+    
+    /// The indexes that have been added in this iteration of the snapshot.
+    var addedIndexes: Set<IndexID> = []
+    
+    /// The indexes that have been completely removed in this iteration of the snapshot.
+    var removedIndexes: Set<IndexID> = []
+    
+    /// The datastore roots that have been added in this iteration of the snapshot.
+    var addedIndexManifests: Set<IndexManifestID> = []
+    
+    /// The datastore roots that have been replaced in this iteration of the snapshot.
+    var removedIndexManifests: Set<IndexManifestID> = []
 }
 
 extension DatastoreRootManifest {
@@ -49,4 +61,27 @@ extension DatastoreRootManifest {
         /// The root object of the index.
         var root: DatastoreIndexManifestIdentifier
     }
+    
+    enum IndexID: Codable, Hashable {
+        case primary
+        case direct(index: DatastoreIndexIdentifier)
+        case secondary(index: DatastoreIndexIdentifier)
+    }
+    
+    enum IndexManifestID: Codable, Hashable {
+        case primary(manifest: DatastoreIndexManifestIdentifier)
+        case direct(index: DatastoreIndexIdentifier, manifest: DatastoreIndexManifestIdentifier)
+        case secondary(index: DatastoreIndexIdentifier, manifest: DatastoreIndexManifestIdentifier)
+        
+        init<AccessMode>(_ id: DiskPersistence<AccessMode>.Datastore.Index.ID) {
+            switch id {
+            case .primary(let manifest):
+                self = .primary(manifest: manifest)
+            case .direct(let index, let manifest):
+                self = .direct(index: index, manifest: manifest)
+            case .secondary(let index, let manifest):
+                self = .secondary(index: index, manifest: manifest)
+            }
+        }
+    }
 }

Sources/CodableDatastore/Persistence/Disk Persistence/DiskPersistence.swift

@@ -462,7 +462,9 @@ extension DiskPersistence {
 
     func persist(
         actionName: String?,
-        roots: [DatastoreKey : Datastore.RootObject]
+        roots: [DatastoreKey : Datastore.RootObject],
+        addedDatastoreRoots: Set<DatastoreRootIdentifier>,
+        removedDatastoreRoots: Set<DatastoreRootIdentifier>
     ) async throws {
         let containsEdits = try await withCurrentSnapshot { snapshot in
             try await snapshot.readingManifest { manifest, iteration in
@@ -485,6 +487,8 @@ extension DiskPersistence {
         try await self.withCurrentSnapshot { snapshot in
             try await snapshot.updatingManifest { manifest, iteration in
                 iteration.actionName = actionName
+                iteration.addedDatastoreRoots = addedDatastoreRoots
+                iteration.removedDatastoreRoots = removedDatastoreRoots
                 for (key, root) in roots {
                     iteration.dataStores[key.rawValue] = SnapshotIteration.DatastoreInfo(
                         key: key,

Sources/CodableDatastore/Persistence/Disk Persistence/Transaction/Transaction.swift

@@ -150,7 +150,15 @@ extension DiskPersistence {
                 try await root.persistIfNeeded()
             }
 
-            try await persistence.persist(actionName: actionName, roots: rootObjects)
+            let addedDatastoreRoots = Set(createdRootObjects.map(\.id))
+            let removedDatastoreRoots = Set(deletedRootObjects.map(\.id))
+            
+            try await persistence.persist(
+                actionName: actionName,
+                roots: rootObjects,
+                addedDatastoreRoots: addedDatastoreRoots,
+                removedDatastoreRoots: removedDatastoreRoots
+            )
 
             var datastores: [DatastoreKey : Datastore] = [:]
             for (datastoreKey, event) in entryMutations {
@@ -324,7 +332,11 @@ extension DiskPersistence.Transaction: DatastoreInterfaceProtocol {
                 rootObject: rootManifest
             )
             createdRootObjects.insert(newRootObject)
-            deletedRootObjects.insert(existingRootObject)
+            if createdRootObjects.contains(existingRootObject) {
+                createdRootObjects.remove(existingRootObject)
+            } else {
+                deletedRootObjects.insert(existingRootObject)
+            }
             await datastore.adopt(rootObject: newRootObject)
             rootObjects[datastoreKey] = newRootObject
         } else {
@@ -360,6 +372,11 @@ extension DiskPersistence.Transaction: DatastoreInterfaceProtocol {
             createdIndexes.insert(primaryIndex)
             await datastore.adopt(index: primaryIndex)
 
+            var addedIndexes: Set<DatastoreRootManifest.IndexID> = []
+            var addedIndexManifests: Set<DatastoreRootManifest.IndexManifestID> = []
+            addedIndexes.insert(.primary)
+            addedIndexManifests.insert(.primary(manifest: primaryManifestIdentifier))
+            
             for indexInfo in directIndexManifests {
                 let index = DiskPersistence.Datastore.Index(
                     datastore: datastore,
@@ -370,6 +387,8 @@ extension DiskPersistence.Transaction: DatastoreInterfaceProtocol {
                     )
                 )
                 createdIndexes.insert(index)
+                addedIndexes.insert(.direct(index: indexInfo.id))
+                addedIndexManifests.insert(.direct(index: indexInfo.id, manifest: indexInfo.root))
                 await datastore.adopt(index: index)
             }
 
@@ -383,20 +402,25 @@ extension DiskPersistence.Transaction: DatastoreInterfaceProtocol {
                     )
                 )
                 createdIndexes.insert(index)
+                addedIndexes.insert(.secondary(index: indexInfo.id))
+                addedIndexManifests.insert(.secondary(index: indexInfo.id, manifest: indexInfo.root))
                 await datastore.adopt(index: index)
             }
 
             var descriptor = descriptor
             descriptor.size = 0
 
+            let modificationDate = Date()
             /// Create the root object from the indexes that were created
             let manifest = DatastoreRootManifest(
-                id: DatastoreRootIdentifier(),
-                modificationDate: Date(),
+                id: DatastoreRootIdentifier(date: modificationDate),
+                modificationDate: modificationDate,
                 descriptor: descriptor,
                 primaryIndexManifest: primaryManifestIdentifier,
                 directIndexManifests: directIndexManifests,
-                secondaryIndexManifests: secondaryIndexManifests
+                secondaryIndexManifests: secondaryIndexManifests,
+                addedIndexes: addedIndexes,
+                addedIndexManifests: addedIndexManifests
             )
 
             let newRoot = DiskPersistence.Datastore.RootObject(
@@ -823,7 +847,11 @@ extension DiskPersistence.Transaction {
             manifest: indexManifest
         )
         createdIndexes.insert(newIndex)
-        deletedIndexes.insert(existingIndex)
+        if createdIndexes.contains(existingIndex) {
+            createdIndexes.insert(existingIndex)
+        } else {
+            deletedIndexes.insert(existingIndex)
+        }
         await datastore.adopt(index: newIndex)
 
         var rootManifest = try await existingRootObject.manifest(replacing: newIndex.id)
@@ -842,7 +870,11 @@ extension DiskPersistence.Transaction {
             rootObject: rootManifest
         )
         createdRootObjects.insert(newRootObject)
-        deletedRootObjects.insert(existingRootObject)
+        if createdRootObjects.contains(existingRootObject) {
+            createdRootObjects.remove(existingRootObject)
+        } else {
+            deletedRootObjects.insert(existingRootObject)
+        }
         await datastore.adopt(rootObject: newRootObject)
         rootObjects[datastoreKey] = newRootObject
     }
@@ -907,7 +939,11 @@ extension DiskPersistence.Transaction {
             manifest: indexManifest
         )
         createdIndexes.insert(newIndex)
-        deletedIndexes.insert(existingIndex)
+        if createdIndexes.contains(existingIndex) {
+            createdIndexes.insert(existingIndex)
+        } else {
+            deletedIndexes.insert(existingIndex)
+        }
         await datastore.adopt(index: newIndex)
 
         var rootManifest = try await existingRootObject.manifest(replacing: newIndex.id)
@@ -926,7 +962,11 @@ extension DiskPersistence.Transaction {
             rootObject: rootManifest
         )
         createdRootObjects.insert(newRootObject)
-        deletedRootObjects.insert(existingRootObject)
+        if createdRootObjects.contains(existingRootObject) {
+            createdRootObjects.remove(existingRootObject)
+        } else {
+            deletedRootObjects.insert(existingRootObject)
+        }
         await datastore.adopt(rootObject: newRootObject)
         rootObjects[datastoreKey] = newRootObject
     }