Merge remote-tracking branch 'origin/develop' into feat/bidirectional…

…_references

costs/src/context.rs

@@ -179,7 +179,7 @@ impl<T> CostsExt for T {}
 /// 1. Early termination on error;
 /// 2. Because of 1, `Result` is removed from the equation;
 /// 3. `CostContext` is removed too because it is added to external cost
-/// accumulator;
+///    accumulator;
 /// 4. Early termination uses external cost accumulator so previous costs won't
 ///    be lost.
 #[macro_export]

grovedb-version/src/lib.rs

@@ -8,13 +8,15 @@ macro_rules! check_grovedb_v0_with_cost {
     ($method:expr, $version:expr) => {{
         const EXPECTED_VERSION: u16 = 0;
         if $version != EXPECTED_VERSION {
-            return Err($crate::error::GroveVersionError::UnknownVersionMismatch {
-                method: $method.to_string(),
-                known_versions: vec![EXPECTED_VERSION],
-                received: $version,
-            }
-            .into())
-            .wrap_with_cost(OperationCost::default());
+            return grovedb_costs::CostsExt::wrap_with_cost(
+                Err($crate::error::GroveVersionError::UnknownVersionMismatch {
+                    method: $method.to_string(),
+                    known_versions: vec![EXPECTED_VERSION],
+                    received: $version,
+                }
+                .into()),
+                Default::default(),
+            );
         }
     }};
 }
@@ -56,13 +58,15 @@ macro_rules! check_merk_v0_with_cost {
     ($method:expr, $version:expr) => {{
         const EXPECTED_VERSION: u16 = 0;
         if $version != EXPECTED_VERSION {
-            return Err($crate::error::GroveVersionError::UnknownVersionMismatch {
-                method: $method.to_string(),
-                known_versions: vec![EXPECTED_VERSION],
-                received: $version,
-            }
-            .into())
-            .wrap_with_cost(OperationCost::default());
+            return grovedb_costs::CostsExt::wrap_with_cost(
+                Err($crate::error::GroveVersionError::UnknownVersionMismatch {
+                    method: $method.to_string(),
+                    known_versions: vec![EXPECTED_VERSION],
+                    received: $version,
+                }
+                .into()),
+                Default::default(),
+            );
         }
     }};
 }

grovedb-version/src/version/v1.rs

@@ -66,7 +66,6 @@ pub const GROVE_V1: GroveVersion = GroveVersion {
             serialize: 0,
             serialized_size: 0,
             deserialize: 0,
-            get_with_value_hash: 0,
             insert_reference_if_changed_value: 0,
         },
         operations: GroveDBOperationsVersions {
@@ -90,7 +89,6 @@ pub const GROVE_V1: GroveVersion = GroveVersion {
                 worst_case_for_get_raw: 0,
                 worst_case_for_get: 0,
                 is_empty_tree: 0,
-                follow_reference_once: 0,
             },
             insert: GroveDBOperationsInsertVersions {
                 insert: 0,

grovedb/Cargo.toml

@@ -34,6 +34,7 @@ tokio = { version = "1.40.0", features = ["rt-multi-thread", "net"], optional =
 tower-http = { version = "0.5.2", features = ["fs"], optional = true }
 zip-extensions = { version = "0.8.1", optional = true }
 serde = { version = "1.0.210", features = ["derive"], optional = true }
+bitvec = { version = "1.0.1", optional = true }
 
 [dev-dependencies]
 grovedb-epoch-based-storage-flags = { version = "3.0.0", path = "../grovedb-epoch-based-storage-flags" }
@@ -57,6 +58,7 @@ full = [
     "minimal",
 ]
 minimal = [
+    "bitvec",
     "grovedb-merk/minimal",
     "thiserror",
     "tempfile",

grovedb/src/batch/mod.rs

@@ -72,7 +72,10 @@
 use crate::{
     batch::{batch_structure::BatchStructure, mode::BatchRunMode},
     bidirectional_references::BidirectionalReference,
-    element::{MaxReferenceHop, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE},
+    element::{
+        MaxReferenceHop, BIG_SUM_TREE_COST_SIZE, COUNT_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE,
+        SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE,
+    },
     operations::{get::MAX_REFERENCE_HOPS, proof::util::hex_to_ascii},
     reference_path::{
         path_from_reference_path_type, path_from_reference_qualified_path_type, ReferencePathType,
@@ -671,19 +674,19 @@
 trait TreeCache<G, SR> {
     fn insert(&mut self, op: &QualifiedGroveDbOp, tree_type: TreeType) -> CostResult<(), Error>;
 
     fn get_batch_run_mode(&self) -> BatchRunMode;
 
     /// We will also be returning an op mode, this is to be used in propagation
     fn execute_ops_on_path(
         &mut self,
         path: &KeyInfoPath,
         ops_at_path_by_key: BTreeMap<KeyInfo, GroveOp>,
         ops_by_qualified_paths: &BTreeMap<Vec<Vec<u8>>, GroveOp>,
         batch_apply_options: &BatchApplyOptions,
         flags_update: &mut G,
         split_removal_bytes: &mut SR,
         grove_version: &GroveVersion,
     ) -> CostResult<RootHashKeyAndAggregateData, Error>;
 
     fn update_base_merk_root_key(
         &mut self,
@@ -737,16 +740,16 @@
     ///   deserializing the referenced element.
     /// * `Error::InvalidBatchOperation`: If the referenced element points to a
     ///   tree being updated.
     fn process_reference<'a, G, SR>(
         &'a mut self,
         qualified_path: &[Vec<u8>],
         ops_by_qualified_paths: &'a BTreeMap<Vec<Vec<u8>>, GroveOp>,
         recursions_allowed: u8,
         intermediate_reference_info: Option<&'a ReferencePathType>,
         flags_update: &mut G,
         split_removal_bytes: &mut SR,
         grove_version: &GroveVersion,
     ) -> CostResult<CryptoHash, Error>
     where
         G: FnMut(&StorageCost, Option<ElementFlags>, &mut ElementFlags) -> Result<bool, Error>,
         SR: FnMut(
@@ -955,17 +958,17 @@
     ///   deserializing the referenced element.
     /// * `Error::InvalidBatchOperation` - If the referenced element points to a
     ///   tree being updated, which is not allowed.
     fn process_reference_with_hop_count_greater_than_one<'a, G, SR>(
         &'a mut self,
         key: &[u8],
         reference_path: &[Vec<u8>],
         qualified_path: &[Vec<u8>],
         ops_by_qualified_paths: &'a BTreeMap<Vec<Vec<u8>>, GroveOp>,
         recursions_allowed: u8,
         flags_update: &mut G,
         split_removal_bytes: &mut SR,
         grove_version: &GroveVersion,
     ) -> CostResult<CryptoHash, Error>
     where
         G: FnMut(&StorageCost, Option<ElementFlags>, &mut ElementFlags) -> Result<bool, Error>,
         SR: FnMut(
@@ -1609,7 +1612,7 @@
                         }
                     };
                     let merk_feature_type =
-                        cost_return_on_error_no_add!(cost, element.get_feature_type(is_sum_tree));
+                        cost_return_on_error_no_add!(cost, element.get_feature_type(in_tree_type));
 
                     cost_return_on_error!(
                         &mut cost,
@@ -2049,27 +2052,27 @@
     /// Method to propagate updated subtree root hashes up to GroveDB root
     /// If the pause height is set in the batch apply options
     /// Then return the list of leftover operations
     fn continue_partial_apply_body<'db, S: StorageContext<'db>>(
         &self,
         previous_leftover_operations: Option<OpsByLevelPath>,
         additional_ops: Vec<QualifiedGroveDbOp>,
         batch_apply_options: Option<BatchApplyOptions>,
         update_element_flags_function: impl FnMut(
             &StorageCost,
             Option<ElementFlags>,
             &mut ElementFlags,
         ) -> Result<bool, Error>,
         split_removed_bytes_function: impl FnMut(
             &mut ElementFlags,
             u32, // key removed bytes
             u32, // value removed bytes
         ) -> Result<
             (StorageRemovedBytes, StorageRemovedBytes),
             Error,
         >,
         get_merk_fn: impl FnMut(&[Vec<u8>], bool) -> CostResult<Merk<S>, Error>,
         grove_version: &GroveVersion,
     ) -> CostResult<Option<OpsByLevelPath>, Error> {
         check_grovedb_v0_with_cost!(
             "continue_partial_apply_body",
             grove_version
@@ -2338,8 +2341,6 @@
             return Ok(()).wrap_with_cost(cost);
         }
 
-        let tx = TxRef::new(&self.db, transaction);
-
         // Determines whether to check batch operation consistency
         // return false if the disable option is set to true, returns true for any other
         // case
@@ -2414,6 +2415,7 @@
         //             .join(" | ")
         //     );
         // }
+
         tx.commit_local().wrap_with_cost(cost)
     }
 
@@ -2421,30 +2423,30 @@
     /// The batch is not committed
     /// Clients should set the Batch Apply Options batch pause height
     /// If it is not set we default to pausing at the root tree
     pub fn apply_partial_batch_with_element_flags_update(
         &self,
         ops: Vec<QualifiedGroveDbOp>,
         batch_apply_options: Option<BatchApplyOptions>,
         mut update_element_flags_function: impl FnMut(
             &StorageCost,
             Option<ElementFlags>,
             &mut ElementFlags,
         ) -> Result<bool, Error>,
         mut split_removal_bytes_function: impl FnMut(
             &mut ElementFlags,
             u32, // key removed bytes
             u32, // value removed bytes
         ) -> Result<
             (StorageRemovedBytes, StorageRemovedBytes),
             Error,
         >,
         mut add_on_operations: impl FnMut(
             &OperationCost,
             &Option<OpsByLevelPath>,
         ) -> Result<Vec<QualifiedGroveDbOp>, Error>,
         transaction: TransactionArg,
         grove_version: &GroveVersion,
     ) -> CostResult<(), Error> {
         check_grovedb_v0_with_cost!(
             "apply_partial_batch_with_element_flags_update",
             grove_version
@@ -2459,8 +2461,6 @@
             return Ok(()).wrap_with_cost(cost);
         }
 
-        let tx = TxRef::new(&self.db, transaction);
-
         let mut batch_apply_options = batch_apply_options.unwrap_or_default();
         if batch_apply_options.batch_pause_height.is_none() {
             // we default to pausing at the root tree, which is the most common case

grovedb/src/bidirectional_references.rs

@@ -12,12 +12,11 @@
     element::{Delta, MaxReferenceHop},
     merk_cache::{MerkCache, MerkHandle},
     operations::insert::InsertOptions,
-    reference_path::ReferencePathType,
-    util::{self, ResolvedReference},
+    reference_path::{follow_reference_once, ReferencePathType, ResolvedReference},
     Element, ElementFlags, Error,
 };
 
 const META_BACKWARD_REFERENCES_PREFIX: &'static [u8] = b"refs";
 
 pub type SlotIdx = usize;
 
@@ -39,7 +38,7 @@
 impl BidirectionalReference {
     /// Given current path removes backward reference from the merk and key
     /// where this bidirectional references points to
     fn remove_backward_reference<'db, 'b, 'c, B: AsRef<[u8]>>(
         self,
         merk_cache: &'c MerkCache<'db, 'b, B>,
         current_path: SubtreePathBuilder<'b, B>,
@@ -53,7 +52,7 @@
             ..
         } = cost_return_on_error!(
             &mut cost,
-            util::follow_reference_once(
+            follow_reference_once(
                 merk_cache,
                 current_path,
                 current_key,
@@ -73,7 +72,7 @@
         Ok(()).wrap_with_cost(cost)
     }
 
     fn remove_backward_reference_resolved<'db, 'c>(
         target_merk: &mut MerkHandle<'db, 'c>,
         target_key: &[u8],
         slot_idx: SlotIdx,
@@ -82,7 +81,7 @@
 
         let (prefix, mut bits) = cost_return_on_error!(
             &mut cost,
             get_backward_references_bitvec(target_merk, &target_key)
         );
 
         bits.set(slot_idx, false);
@@ -112,7 +111,7 @@
 
 /// Insert bidirectional reference at specified location performing required
 /// checks and updates
 pub(crate) fn process_bidirectional_reference_insertion<'db, 'b, 'k, B: AsRef<[u8]>>(
     merk_cache: &MerkCache<'db, 'b, B>,
     path: SubtreePath<'b, B>,
     key: &[u8],
@@ -131,7 +130,7 @@
         ..
     } = cost_return_on_error!(
         &mut cost,
-        util::follow_reference_once(
+        follow_reference_once(
             merk_cache,
             path.derive_owned(),
             key,
@@ -219,7 +218,7 @@
                 key,
                 target_node_value_hash,
                 options.map(|o| o.as_merk_options()),
                 &merk_cache.version,
             )
         })
     );
@@ -277,7 +276,7 @@
 
 /// Post-processing of possible backward references relationships after
 /// insertion of anything but bidirectional reference
 pub(crate) fn process_update_element_with_backward_references<'db, 'b, 'c, B: AsRef<[u8]>>(
     merk_cache: &'c MerkCache<'db, 'b, B>,
     merk: MerkHandle<'db, 'c>,
     path: SubtreePath<'b, B>,
@@ -310,7 +309,7 @@
                     merk,
                     path.derive_owned(),
                     key.to_vec(),
                     cost_return_on_error!(&mut cost, delta.new.value_hash(&merk_cache.version))
                 )
             );
         }
@@ -346,7 +345,7 @@
                     merk,
                     path.derive_owned(),
                     key.to_vec(),
                     cost_return_on_error!(&mut cost, delta.new.value_hash(&merk_cache.version))
                 )
             );
 
@@ -388,7 +387,7 @@
 
 /// Recursively deletes all backward references' chains of a key if all of them
 /// allow cascade deletion.
 fn delete_backward_references_recursively<'db, 'b, 'c, B: AsRef<[u8]>>(
     merk_cache: &'c MerkCache<'db, 'b, B>,
     merk: MerkHandle<'db, 'c>,
     path: SubtreePathBuilder<'b, B>,
@@ -422,7 +421,7 @@
                 ..
             } = cost_return_on_error!(
                 &mut cost,
-                util::follow_reference_once(
+                follow_reference_once(
                     merk_cache,
                     current_path.clone(),
                     &current_key,
@@ -444,24 +443,25 @@
         }
 
         // ... and the element altogether
-        cost_return_on_error!(
-            &mut cost,
-            current_merk.for_merk(|m| Element::delete(
-                m,
-                current_key,
-                None,
-                false,
-                false,
-                &merk_cache.version
-            ))
-        );
+        todo!()
+        // cost_return_on_error!(
+        //     &mut cost,
+        //     current_merk.for_merk(|m| Element::delete_with_sectioned_removal_bytes(
+        //         m,
+        //         current_key,
+        //         None,
+        //         false,
+        //         false,
+        //         &merk_cache.version
+        //     ))
+        // );
     }
 
     Ok(()).wrap_with_cost(cost)
 }
 
 /// Recursively updates all backward references' chains of a key.
 fn propagate_backward_references<'db, 'b, 'c, B: AsRef<[u8]>>(
     merk_cache: &'c MerkCache<'db, 'b, B>,
     merk: MerkHandle<'db, 'c>,
     path: SubtreePathBuilder<'b, B>,
@@ -487,7 +487,7 @@
                 ..
             } = cost_return_on_error!(
                 &mut cost,
-                util::follow_reference_once(
+                follow_reference_once(
                     merk_cache,
                     current_path.clone(),
                     &current_key,
@@ -522,7 +522,7 @@
 }
 
 impl BackwardReference {
     fn serialize(&self) -> Result<Vec<u8>, Error> {
         let config = config::standard().with_big_endian().with_no_limit();
         bincode::encode_to_vec(self, config).map_err(|e| {
             Error::CorruptedData(format!("unable to serialize backward reference {}", e))
@@ -537,9 +537,9 @@
     }
 }
 
 type Prefix = Vec<u8>;
 
 fn make_meta_prefix(key: &[u8]) -> Vec<u8> {
     let mut backrefs_for_key = META_BACKWARD_REFERENCES_PREFIX.to_vec();
     backrefs_for_key.extend_from_slice(&key.len().to_be_bytes());
     backrefs_for_key.extend_from_slice(key);
@@ -551,7 +551,7 @@
 /// Prefix for a Merk's meta storage is made of constant keyword, lenght of the
 /// key and the key itself. Under the prefix GroveDB stores bitvec, and slots
 /// for backward references are integers appended to the prefix.
 fn get_backward_references_bitvec<'db, 'c>(
     merk: &mut MerkHandle<'db, 'c>,
     key: &[u8],
 ) -> CostResult<(Prefix, BitArray<[u32; 1], Lsb0>), Error> {
@@ -590,7 +590,7 @@
 /// bitvec to locate them. That way the bits is stored under
 /// [META_BACKWARD_REFERENCES_PREFIX] with key and references themselves are
 /// located under this prefix with index (0-31) appended.
 fn add_backward_reference<'db, 'c>(
     target_merk: &mut MerkHandle<'db, 'c>,
     key: &[u8],
     backward_reference: BackwardReference,
@@ -631,7 +631,7 @@
 }
 
 /// Return a vector of backward references to the item
 fn get_backward_references<'db, 'c>(
     merk: &mut MerkHandle<'db, 'c>,
     key: &[u8],
 ) -> CostResult<Vec<(SlotIdx, BackwardReference)>, Error> {

grovedb/src/debugger.rs

@@ -228,10 +228,15 @@
     }): Json<WithSession<NodeFetchRequest>>,
 ) -> Result<Json<Option<NodeUpdate>>, AppError> {
     let db = state.get_snapshot(session_id).await?;
-    let tx = db.start_transaction();
+    let transaction = db.start_transaction();
 
     let merk = db
-        .open_transactional_merk_at_path(path.as_slice().into(), &tx, None, GroveVersion::latest())
+        .open_transactional_merk_at_path(
+            path.as_slice().into(),
+            &transaction,
+            None,
+            GroveVersion::latest(),
+        )
         .unwrap()?;
     let node = merk.get_node_dbg(&key)?;
 
@@ -251,10 +256,15 @@
     }): Json<WithSession<()>>,
 ) -> Result<Json<Option<NodeUpdate>>, AppError> {
     let db = state.get_snapshot(session_id).await?;
-    let tx = db.start_transaction();
+    let transaction = db.start_transaction();
 
     let merk = db
-        .open_transactional_merk_at_path(SubtreePath::empty(), &tx, None, GroveVersion::latest())
+        .open_transactional_merk_at_path(
+            SubtreePath::empty(),
+            &transaction,
+            None,
+            GroveVersion::latest(),
+        )
         .unwrap()?;
 
     let node = merk.get_root_node_dbg()?;
@@ -321,7 +331,6 @@
     query_result: QueryResultElements,
 ) -> Result<Vec<NodeUpdate>, crate::Error> {
     let mut result = Vec::new();
-    let transaction = db.start_transaction();
 
     let mut last_merk: Option<(Vec<Vec<u8>>, grovedb_merk::Merk<_>)> = None;
 
@@ -334,7 +343,7 @@
                         path.clone(),
                         db.open_transactional_merk_at_path(
                             path.as_slice().into(),
                             &tx,
                             None,
                             GroveVersion::latest(),
                         )

grovedb/src/element/constructor.rs

@@ -78,20 +78,20 @@ impl Element {
         Element::Item(item_value, None)
     }
 
-    #[cfg(feature = "full")]
+    #[cfg(feature = "minimal")]
     /// Set element to an item without flags that allows for a backwards
     /// reference
     pub fn new_item_allowing_bidirectional_references(item_value: Vec<u8>) -> Self {
         Element::ItemWithBackwardsReferences(item_value, None)
     }
 
-    #[cfg(feature = "full")]
+    #[cfg(feature = "minimal")]
     /// Set element to an item with flags
     pub fn new_item_with_flags(item_value: Vec<u8>, flags: Option<ElementFlags>) -> Self {
         Element::Item(item_value, flags)
     }
 
-    #[cfg(feature = "full")]
+    #[cfg(feature = "minimal")]
     /// Set element to an item with flags and that allows for a backwards
     /// reference
     pub fn new_item_allowing_bidirectional_references_with_flags(
@@ -101,26 +101,26 @@ impl Element {
         Element::ItemWithBackwardsReferences(item_value, flags)
     }
 
-    #[cfg(feature = "full")]
+    #[cfg(feature = "minimal")]
     /// Set element to a sum item without flags
     pub fn new_sum_item(value: i64) -> Self {
         Element::SumItem(value, None)
     }
 
-    #[cfg(feature = "full")]
+    #[cfg(feature = "minimal")]
     /// Set element to a sum item hat allows for backwards references without
     /// flags
     pub fn new_sum_item_allowing_bidirectional_references(value: i64) -> Self {
         Element::SumItemWithBackwardsReferences(value, None)
     }
 
-    #[cfg(feature = "full")]
+    #[cfg(feature = "minimal")]
     /// Set element to a sum item with flags
     pub fn new_sum_item_with_flags(value: i64, flags: Option<ElementFlags>) -> Self {
         Element::SumItem(value, flags)
     }
 
-    #[cfg(feature = "full")]
+    #[cfg(feature = "minimal")]
     /// Set element to a sum item hat allows for backwards references with flags
     pub fn new_sum_item_allowing_bidirectional_references_with_flags(
         value: i64,
@@ -129,7 +129,6 @@ impl Element {
         Element::SumItemWithBackwardsReferences(value, flags)
     }
 
-    #[cfg(feature = "full")]
     /// Set element to a reference without flags
     pub fn new_reference(reference_path: ReferencePathType) -> Self {
         Element::Reference(reference_path, None, None)

grovedb/src/element/delete.rs

@@ -11,58 +11,13 @@ use grovedb_merk::{BatchEntry, Error as MerkError, Merk, MerkOptions, Op};
 use grovedb_storage::StorageContext;
 #[cfg(feature = "minimal")]
 use grovedb_version::check_grovedb_v0_with_cost;
-#[cfg(feature = "full")]
+#[cfg(feature = "minimal")]
 use grovedb_version::version::GroveVersion;
 
 #[cfg(feature = "minimal")]
 use crate::{Element, Error};
 
 impl Element {
-    #[cfg(feature = "minimal")]
-    /// Delete an element from Merk under a key
-    pub fn delete<'db, K: AsRef<[u8]>, S: StorageContext<'db>>(
-        merk: &mut Merk<S>,
-        key: K,
-        merk_options: Option<MerkOptions>,
-        is_layered: bool,
-        in_tree_type: TreeType,
-        grove_version: &GroveVersion,
-    ) -> CostResult<(), Error> {
-        check_grovedb_v0_with_cost!("delete", grove_version.grovedb_versions.element.delete);
-        let op = match (in_tree_type, is_layered) {
-            (TreeType::NormalTree, true) => Op::DeleteLayered,
-            (TreeType::NormalTree, false) => Op::Delete,
-            (TreeType::SumTree, true)
-            | (TreeType::BigSumTree, true)
-            | (TreeType::CountTree, true)
-            | (TreeType::CountSumTree, true) => Op::DeleteLayeredMaybeSpecialized,
-            (TreeType::SumTree, false)
-            | (TreeType::BigSumTree, false)
-            | (TreeType::CountTree, false)
-            | (TreeType::CountSumTree, false) => Op::DeleteMaybeSpecialized,
-        };
-        let batch = [(key, op)];
-        // todo not sure we get it again, we need to see if this is necessary
-        let tree_type = merk.tree_type;
-        merk.apply_with_specialized_costs::<_, Vec<u8>>(
-            &batch,
-            &[],
-            merk_options,
-            &|key, value| {
-                Self::specialized_costs_for_key_value(
-                    key,
-                    value,
-                    tree_type.inner_node_type(),
-                    grove_version,
-                )
-                .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))
-            },
-            Some(&Element::value_defined_cost_for_serialized_value),
-            grove_version,
-        )
-        .map_err(|e| Error::CorruptedData(e.to_string()))
-    }
-
     #[cfg(feature = "minimal")]
     /// Delete an element from Merk under a key
     pub fn delete_with_sectioned_removal_bytes<'db, K: AsRef<[u8]>, S: StorageContext<'db>>(