Added disk persistence support for removing indexes

Author: dimitribouniol

README.md

@@ -45,19 +45,12 @@ targets: [
 `CodableDatastore` is a collection of types that make it easy to interface with large data stores of independent types without loading the entire data store in memory.
 
 > **Warning**
-> DO NOT USE THIS IN PRODUCTION PROJECTS. As this project is currently still in its alpha phase, I cannot stress how important it is to not ship anything that relies on this code, or you will experience data loss. There is a chance the underlying model may continue to change day to day, or I will not be able to ever finish it.
-> Until then, please enjoy the code as a spectator or play around with it in toy projects to submit feedback!
-
-### Road to 0.1 Betas
-
-As this project matures towards its first beta, a number of features still need to be fleshed out:
-- Index deletion
-
-The above list will be kept up to date during development and will likely see additions during that process.
+> THINK CAREFULLY ABOUT USING THIS IN PRODUCTION PROJECTS. As this project only just entered its beta phase, I cannot stress how important it is to be very careful about shipping anything that relies on this code, as you may experience data loss migrating to a newer version. Although less likely, there is a chance the underlying model may change in an incompatible way that is not worth supporting with migrations.
+> Until then, please enjoy the code as a spectator or play around with it in toy projects to submit feedback! If you would like to be notified when `CodableDatastore` enters a production-ready state, please follow [#CodableDatastore](https://mastodon.social/tags/CodableDatastore) on Mastodon.
 
 ### Road to 1.0
 
-Once an initial beta is released, the project will start focussing on the functionality and work listed below:
+As this project matures towards release, the project will focus on the functionality and work listed below:
 - Force migration methods
 - Composite indexes (via macros?)
 - Cleaning up old resources in memory
@@ -71,6 +64,8 @@ Once an initial beta is released, the project will start focussing on the functi
 - A memory persistence useful for testing apps with
 - A pre-configured data store tuned to storing pure Data, useful for types like Images
 
+The above list will be kept up to date during development and will likely see additions during that process.
+
 ### Beyond 1.0
 
 Once the 1.0 release has been made, it'll be time to start working on additional features:

Sources/CodableDatastore/Datastore/Dictionary+RawRepresentable.swift

@@ -22,4 +22,10 @@ extension Dictionary {
             yield &self[key.rawValue]
         }
     }
+    
+    @discardableResult
+    @usableFromInline
+    mutating func removeValue(forKey key: some RawRepresentable<Key>) -> Value? {
+        removeValue(forKey: key.rawValue)
+    }
 }

Sources/CodableDatastore/Persistence/Disk Persistence/Datastore/DatastoreIndex.swift

@@ -1399,11 +1399,51 @@ extension DiskPersistence.Datastore.Index {
             }
         }
 
-        if !createdPages.isEmpty {
+        if !createdPages.isEmpty || !removedPages.isEmpty {
             manifest.id = newIndexID.manifestID
             manifest.orderedPages = newOrderedPages
         }
 
         return (manifest: manifest, createdPages: createdPages, removedPages: removedPages)
     }
+    
+    func manifestDeletingAllEntries() async throws -> (
+        manifest: DatastoreIndexManifest,
+        removedPages: Set<DiskPersistence.Datastore.Page>
+    ) {
+        var manifest = try await manifest
+        
+        let newIndexID = id.with(manifestID: DatastoreIndexManifestIdentifier())
+        var removedPages: Set<DiskPersistence.Datastore.Page> = []
+        
+        let originalOrderedPages = manifest.orderedPages
+        var newOrderedPages: [DatastoreIndexManifest.PageInfo] = []
+        newOrderedPages.reserveCapacity(originalOrderedPages.count)
+        
+        for pageInfo in originalOrderedPages {
+            switch pageInfo {
+            case .removed:
+                /// Skip previously removed entries, unless this index is based on a transient index, and the removed entry was from before the transaction began.
+                if !isPersisted {
+                    newOrderedPages.append(pageInfo)
+                }
+                continue
+            case .existing(let pageID), .added(let pageID):
+                let existingPage = await datastore.page(for: .init(index: self.id, page: pageID))
+                /// If the index had data on disk, or it existed prior to the transaction, mark it as removed.
+                /// Otherwise, simply skip the page, since we added it in a transient index.
+                removedPages.insert(existingPage)
+                if isPersisted || pageInfo.isExisting {
+                    newOrderedPages.append(.removed(pageID))
+                }
+            }
+        }
+        
+        if !removedPages.isEmpty {
+            manifest.id = newIndexID.manifestID
+            manifest.orderedPages = newOrderedPages
+        }
+        
+        return (manifest: manifest, removedPages: removedPages)
+    }
 }

Sources/CodableDatastore/Persistence/Disk Persistence/Datastore/DatastoreRoot.swift

@@ -304,5 +304,62 @@ extension DiskPersistence.Datastore.RootObject {
         }
         return updatedManifest
     }
+    
+    func manifest(
+        deleting index: DiskPersistence.Datastore.Index.ID
+    ) async throws -> DatastoreRootManifest {
+        let manifest = try await manifest
+        var updatedManifest = manifest
+        
+        var removedIndex: DatastoreRootManifest.IndexManifestID
+        var addedIndex: DatastoreRootManifest.IndexManifestID?
+        
+        switch index {
+        case .primary(let manifestID):
+            removedIndex = .primary(manifest: manifestID)
+            /// Primary index must have _a_ root, so make a new one.
+            let newManifestID = DatastoreIndexManifestIdentifier()
+            addedIndex = .primary(manifest: newManifestID)
+            updatedManifest.primaryIndexManifest = newManifestID
+        case .direct(let indexID, let manifestID):
+            removedIndex = .direct(index: indexID, manifest: manifestID)
+            if let entryIndex = updatedManifest.directIndexManifests.firstIndex(where: { $0.id == indexID }) {
+                let indexName = updatedManifest.directIndexManifests[entryIndex].key
+                updatedManifest.directIndexManifests.remove(at: entryIndex)
+                updatedManifest.descriptor.directIndexes.removeValue(forKey: indexName)
+            }
+        case .secondary(let indexID, let manifestID):
+            removedIndex = .secondary(index: indexID, manifest: manifestID)
+            if let entryIndex = updatedManifest.secondaryIndexManifests.firstIndex(where: { $0.id == indexID }) {
+                let indexName = updatedManifest.secondaryIndexManifests[entryIndex].key
+                updatedManifest.secondaryIndexManifests.remove(at: entryIndex)
+                updatedManifest.descriptor.secondaryIndexes.removeValue(forKey: indexName)
+            }
+        }
+        
+        if manifest != updatedManifest {
+            let modificationDate = Date()
+            updatedManifest.id = DatastoreRootIdentifier(date: modificationDate)
+            updatedManifest.modificationDate = modificationDate
+            
+            if isPersisted {
+                updatedManifest.removedIndexes = []
+                updatedManifest.removedIndexManifests = []
+                updatedManifest.addedIndexes = []
+                updatedManifest.addedIndexManifests = []
+            }
+            
+            if updatedManifest.addedIndexManifests.contains(removedIndex) {
+                updatedManifest.addedIndexManifests.remove(removedIndex)
+            } else {
+                updatedManifest.removedIndexManifests.insert(removedIndex)
+            }
+            
+            if let addedIndex {
+                updatedManifest.addedIndexManifests.insert(addedIndex)
+            }
+        }
+        return updatedManifest
+    }
 }
 

Sources/CodableDatastore/Persistence/Disk Persistence/Transaction/Transaction.swift

@@ -1003,7 +1003,54 @@ extension DiskPersistence.Transaction {
     ) async throws {
         try checkIsActive()
 
-        preconditionFailure("Unimplemented")
+        guard let existingRootObject = try await rootObject(for: datastoreKey)
+        else { throw DatastoreInterfaceError.datastoreKeyNotFound }
+        
+        let existingIndex = try await existingRootObject.primaryIndex
+        
+        let datastore = existingRootObject.datastore
+        
+        let (indexManifest, removedPages) = try await existingIndex.manifestDeletingAllEntries()
+        
+        /// No change occured, bail early
+        guard existingIndex.id.manifestID != indexManifest.id else { return }
+        
+        deletedPages.formUnion(removedPages)
+        
+        let newIndex = DiskPersistence.Datastore.Index(
+            datastore: datastore,
+            id: existingIndex.id.with(manifestID: indexManifest.id),
+            manifest: indexManifest
+        )
+        createdIndexes.insert(newIndex)
+        if createdIndexes.contains(existingIndex) {
+            createdIndexes.insert(existingIndex)
+        } else {
+            deletedIndexes.insert(existingIndex)
+        }
+        await datastore.adopt(index: newIndex)
+        
+        var rootManifest = try await existingRootObject.manifest(replacing: newIndex.id)
+        
+        /// Reset the number of entries we are managing.
+        rootManifest.descriptor.size = 0
+        
+        /// No change occured, bail early
+        guard existingRootObject.id != rootManifest.id else { return }
+        
+        let newRootObject = DiskPersistence.Datastore.RootObject(
+            datastore: existingRootObject.datastore,
+            id: rootManifest.id,
+            rootObject: rootManifest
+        )
+        createdRootObjects.insert(newRootObject)
+        if createdRootObjects.contains(existingRootObject) {
+            createdRootObjects.remove(existingRootObject)
+        } else {
+            deletedRootObjects.insert(existingRootObject)
+        }
+        await datastore.adopt(rootObject: newRootObject)
+        rootObjects[datastoreKey] = newRootObject
     }
 
     func persistDirectIndexEntry<IndexType: Indexable, IdentifierType: Indexable>(
@@ -1060,7 +1107,34 @@ extension DiskPersistence.Transaction {
     ) async throws {
         try checkIsActive()
 
-        preconditionFailure("Unimplemented")
+        guard let existingRootObject = try await rootObject(for: datastoreKey)
+        else { throw DatastoreInterfaceError.datastoreKeyNotFound }
+        
+        guard let existingIndex = try await existingRootObject.directIndexes[indexName]
+        else { throw DatastoreInterfaceError.indexNotFound }
+        
+        let datastore = existingRootObject.datastore
+        
+        deletedIndexes.insert(existingIndex)
+        
+        let rootManifest = try await existingRootObject.manifest(deleting: existingIndex.id)
+        
+        /// No change occured, bail early
+        guard existingRootObject.id != rootManifest.id else { return }
+        
+        let newRootObject = DiskPersistence.Datastore.RootObject(
+            datastore: existingRootObject.datastore,
+            id: rootManifest.id,
+            rootObject: rootManifest
+        )
+        createdRootObjects.insert(newRootObject)
+        if createdRootObjects.contains(existingRootObject) {
+            createdRootObjects.remove(existingRootObject)
+        } else {
+            deletedRootObjects.insert(existingRootObject)
+        }
+        await datastore.adopt(rootObject: newRootObject)
+        rootObjects[datastoreKey] = newRootObject
     }
 
     func persistSecondaryIndexEntry<IndexType: Indexable, IdentifierType: Indexable>(
@@ -1113,7 +1187,34 @@ extension DiskPersistence.Transaction {
     ) async throws {
         try checkIsActive()
 
-        preconditionFailure("Unimplemented")
+        guard let existingRootObject = try await rootObject(for: datastoreKey)
+        else { throw DatastoreInterfaceError.datastoreKeyNotFound }
+        
+        guard let existingIndex = try await existingRootObject.secondaryIndexes[indexName]
+        else { throw DatastoreInterfaceError.indexNotFound }
+        
+        let datastore = existingRootObject.datastore
+        
+        deletedIndexes.insert(existingIndex)
+        
+        let rootManifest = try await existingRootObject.manifest(deleting: existingIndex.id)
+        
+        /// No change occured, bail early
+        guard existingRootObject.id != rootManifest.id else { return }
+        
+        let newRootObject = DiskPersistence.Datastore.RootObject(
+            datastore: existingRootObject.datastore,
+            id: rootManifest.id,
+            rootObject: rootManifest
+        )
+        createdRootObjects.insert(newRootObject)
+        if createdRootObjects.contains(existingRootObject) {
+            createdRootObjects.remove(existingRootObject)
+        } else {
+            deletedRootObjects.insert(existingRootObject)
+        }
+        await datastore.adopt(rootObject: newRootObject)
+        rootObjects[datastoreKey] = newRootObject
     }
 }