Feature/string compression

CHANGELOG.md

@@ -17,7 +17,7 @@
 -----------
 
 ### Internals
-* None.
+* Ability to enumerate a string column has been removed.
 
 ----------------------------------------------
 

src/realm/CMakeLists.txt

@@ -62,6 +62,8 @@ set(REALM_SOURCES
     table.cpp
     table_ref.cpp
     obj_list.cpp
+    string_interner.cpp
+    string_compressor.cpp
     object_id.cpp
     table_view.cpp
     tokenizer.cpp
@@ -178,6 +180,8 @@ set(REALM_INSTALL_HEADERS
     null.hpp
     obj.hpp
     obj_list.hpp
+    string_interner.hpp
+    string_compressor.hpp
     object_id.hpp
     path.hpp
     owned_data.hpp

src/realm/array.cpp

@@ -294,7 +294,7 @@ void Array::set_type(Type type)
     set_hasrefs_in_header(init_has_refs, header);
 }
 
-void Array::destroy_children(size_t offset) noexcept
+void Array::destroy_children(size_t offset, bool ro_only) noexcept
 {
     for (size_t i = offset; i != m_size; ++i) {
         int64_t value = get(i);
@@ -310,22 +310,10 @@ void Array::destroy_children(size_t offset) noexcept
             continue;
 
         ref_type ref = to_ref(value);
-        destroy_deep(ref, m_alloc);
+        destroy_deep(ref, m_alloc, ro_only);
     }
 }
 
-// size_t Array::get_byte_size() const noexcept
-//{
-//     const auto header = get_header();
-//     auto num_bytes = get_byte_size_from_header(header);
-//     auto read_only = m_alloc.is_read_only(m_ref) == true;
-//     auto capacity = get_capacity_from_header(header);
-//     auto bytes_ok = num_bytes <= capacity;
-//     REALM_ASSERT(read_only || bytes_ok);
-//     REALM_ASSERT_7(m_alloc.is_read_only(m_ref), ==, true, ||, num_bytes, <=, get_capacity_from_header(header));
-//     return num_bytes;
-// }
-
 ref_type Array::do_write_shallow(_impl::ArrayWriterBase& out) const
 {
     // here we might want to compress the array and write down.

src/realm/array.hpp

@@ -117,7 +117,7 @@ class Array : public Node, public ArrayParent {
     /// pointer.
     void init_from_mem(MemRef) noexcept;
 
-    /// Same as `init_from_ref(get_ref_from_parent())`.
+    /// Same as `init_from_ref(ref_from_parent())`.
     void init_from_parent() noexcept
     {
         ref_type ref = get_ref_from_parent();
@@ -362,7 +362,8 @@ class Array : public Node, public ArrayParent {
     /// state (as if calling detach()), then free the allocated memory. If this
     /// accessor is already in the detached state, this function has no effect
     /// (idempotency).
-    void destroy_deep() noexcept;
+    /// If 'ro_only', only free space in read-only memory (the file)
+    void destroy_deep(bool ro_only = false) noexcept;
 
     /// check if the array is encoded (in B format)
     inline bool is_compressed() const;
@@ -377,13 +378,13 @@ class Array : public Node, public ArrayParent {
     bool try_decompress();
 
     /// Shorthand for `destroy_deep(MemRef(ref, alloc), alloc)`.
-    static void destroy_deep(ref_type ref, Allocator& alloc) noexcept;
+    static void destroy_deep(ref_type ref, Allocator& alloc, bool ro_only = false) noexcept;
 
     /// Destroy the specified array node and all of its children, recursively.
     ///
     /// This is done by freeing the specified array node after calling
     /// destroy_deep() for every contained 'ref' element.
-    static void destroy_deep(MemRef, Allocator&) noexcept;
+    static void destroy_deep(MemRef, Allocator&, bool ro_only = false) noexcept;
 
     // Clone deep
     static MemRef clone(MemRef, Allocator& from_alloc, Allocator& target_alloc);
@@ -544,7 +545,7 @@ class Array : public Node, public ArrayParent {
     // Overriding method in ArrayParent
     ref_type get_child_ref(size_t) const noexcept override;
 
-    void destroy_children(size_t offset = 0) noexcept;
+    void destroy_children(size_t offset = 0, bool ro_only = false) noexcept;
 
 protected:
     // Getters and Setters for adaptive-packed arrays
@@ -916,16 +917,17 @@ inline void Array::set_context_flag(bool value) noexcept
     }
 }
 
-inline void Array::destroy_deep() noexcept
+inline void Array::destroy_deep(bool ro_only) noexcept
 {
     if (!is_attached())
         return;
 
     if (m_has_refs)
-        destroy_children();
+        destroy_children(0, ro_only);
 
     char* header = get_header_from_data(m_data);
-    m_alloc.free_(m_ref, header);
+    if (!ro_only || is_read_only())
+        m_alloc.free_(m_ref, header);
     m_data = nullptr;
 }
 
@@ -968,20 +970,21 @@ inline void Array::clear_and_destroy_children()
     truncate_and_destroy_children(0);
 }
 
-inline void Array::destroy_deep(ref_type ref, Allocator& alloc) noexcept
+inline void Array::destroy_deep(ref_type ref, Allocator& alloc, bool ro_only) noexcept
 {
-    destroy_deep(MemRef(ref, alloc), alloc);
+    destroy_deep(MemRef(ref, alloc), alloc, ro_only);
 }
 
-inline void Array::destroy_deep(MemRef mem, Allocator& alloc) noexcept
+inline void Array::destroy_deep(MemRef mem, Allocator& alloc, bool ro_only) noexcept
 {
     if (!get_hasrefs_from_header(mem.get_addr())) {
-        alloc.free_(mem);
+        if (!ro_only || alloc.is_read_only(mem.get_ref()))
+            alloc.free_(mem);
         return;
     }
     Array array(alloc);
     array.init_from_mem(mem);
-    array.destroy_deep();
+    array.destroy_deep(ro_only);
 }
 
 

src/realm/array_backlink.cpp

@@ -225,12 +225,12 @@ void ArrayBacklink::verify() const
                         REALM_ASSERT(src_obj.get<Mixed>(src_col_key).get_link() == target_link);
                     }
                     else if (val.is_type(type_List)) {
-                        DummyParent parent(src_table, val.get_ref());
+                        DummyParent parent(src_table, val.get_ref(), src_col_key);
                         Lst<Mixed> list(parent, 0);
                         REALM_ASSERT(list.find_any(target_link) != npos);
                     }
                     else if (val.is_type(type_Dictionary)) {
-                        DummyParent parent(src_table, val.get_ref());
+                        DummyParent parent(src_table, val.get_ref(), src_col_key);
                         Dictionary dict(parent, 0);
                         REALM_ASSERT(dict.find_any(target_link) != npos);
                     }

src/realm/array_integer.cpp

@@ -22,6 +22,8 @@
 #include <realm/impl/destroy_guard.hpp>
 #include <realm/column_integer.hpp>
 
+#include <iostream>
+
 using namespace realm;
 
 ArrayInteger::ArrayInteger(Allocator& allocator) noexcept

src/realm/array_integer.hpp

@@ -176,6 +176,7 @@ inline ArrayIntNull::~ArrayIntNull() noexcept {}
 
 inline size_t ArrayIntNull::size() const noexcept
 {
+    // this cannot be right, what if size is 0
     return Array::size() - 1;
 }
 

src/realm/array_mixed.cpp

@@ -118,6 +118,22 @@ void ArrayMixed::set_null(size_t ndx)
     }
 }
 
+std::optional<StringID> ArrayMixed::get_string_id(size_t ndx) const
+{
+    int64_t val = m_composite.get(ndx);
+    if (val) {
+        const int64_t int_val = val >> s_data_shift;
+        const size_t payload_ndx{(size_t)int_val};
+        const DataType type((val & s_data_type_mask) - 1);
+        if (type == type_String) {
+            ensure_string_array();
+            REALM_ASSERT(size_t(int_val) < m_strings.size());
+            return m_strings.get_string_id(payload_ndx);
+        }
+    }
+    return {};
+}
+
 Mixed ArrayMixed::get(size_t ndx) const
 {
     int64_t val = m_composite.get(ndx);
@@ -363,9 +379,8 @@ ref_type ArrayMixed::typed_write(ref_type top_ref, _impl::ArrayWriterBase& out,
      2. int and pair int arrays, they are used for storing integers, timestamps, floats, doubles,
      decimals, links. In general we can compress them, but we need to be careful, controlling the col_type
      should prevent compressing data that we want to leave in the current format.
-     3. string array is for strings and binary data (no compression for now)
-     4. ref array is actually storing refs to collections. they can only be BPlusTree<int, Mixed> or
-     BPlusTree<string, Mixed>.
+     3. string array is for strings and binary data
+     4. ref array is actually storing refs to collections. They can only be Lst<Mixed> or Dictionary.
      5. key array stores unique identifiers for collections in mixed (integers that can be compressed)
      */
     Array composite(alloc);
@@ -375,41 +390,48 @@ ref_type ArrayMixed::typed_write(ref_type top_ref, _impl::ArrayWriterBase& out,
         auto ref = top.get(i);
         ref_type new_ref = ref;
         if (ref && !(out.only_modified && alloc.is_read_only(ref))) {
-            if (i < 3) { // int, and pair_int
-                // integer arrays
-                new_ref = Array::write(ref, alloc, out, out.only_modified, out.compress);
-            }
-            else if (i == 4) { // collection in mixed
-                ArrayRef arr_ref(alloc);
-                arr_ref.init_from_ref(ref);
-                auto ref_sz = arr_ref.size();
-                TempArray written_ref_leaf(ref_sz);
-
-                for (size_t k = 0; k < ref_sz; k++) {
-                    ref_type new_sub_ref = 0;
-                    if (auto sub_ref = arr_ref.get(k)) {
-                        auto header = alloc.translate(sub_ref);
-                        // Now we have to find out if the nested collection is a
-                        // dictionary or a list. If the top array has a size of 2
-                        // and it is not a BplusTree inner node, then it is a dictionary
-                        if (NodeHeader::get_size_from_header(header) == 2 &&
-                            !NodeHeader::get_is_inner_bptree_node_from_header(header)) {
-                            new_sub_ref = Dictionary::typed_write(sub_ref, out, alloc);
-                        }
-                        else {
-                            new_sub_ref = BPlusTree<Mixed>::typed_write(sub_ref, out, alloc);
+            switch (i) {
+                case payload_idx_int:
+                    // integer array
+                    new_ref = Array::write(ref, alloc, out, out.only_modified, out.compress);
+                    break;
+                case payload_idx_pair:
+                    // integer array
+                    new_ref = Array::write(ref, alloc, out, out.only_modified, out.compress);
+                    break;
+                case payload_idx_str:
+                    new_ref = ArrayString::typed_write(ref, out, alloc);
+                    break;
+                case payload_idx_ref: {
+                    // collection in mixed
+                    ArrayRef arr_ref(alloc);
+                    arr_ref.init_from_ref(ref);
+                    auto ref_sz = arr_ref.size();
+                    TempArray written_ref_leaf(ref_sz);
+
+                    for (size_t k = 0; k < ref_sz; k++) {
+                        ref_type new_sub_ref = 0;
+                        if (auto sub_ref = arr_ref.get(k)) {
+                            auto header = alloc.translate(sub_ref);
+                            // Now we have to find out if the nested collection is a
+                            // dictionary or a list. If the top array has a size of 2
+                            // and it is not a BplusTree inner node, then it is a dictionary
+                            if (NodeHeader::get_size_from_header(header) == 2 &&
+                                !NodeHeader::get_is_inner_bptree_node_from_header(header)) {
+                                new_sub_ref = Dictionary::typed_write(sub_ref, out, alloc);
+                            }
+                            else {
+                                new_sub_ref = BPlusTree<Mixed>::typed_write(sub_ref, out, alloc);
+                            }
                         }
+                        written_ref_leaf.set_as_ref(k, new_sub_ref);
                     }
-                    written_ref_leaf.set_as_ref(k, new_sub_ref);
+                    new_ref = written_ref_leaf.write(out);
+                    break;
                 }
-                new_ref = written_ref_leaf.write(out);
-            }
-            else if (i == 5) { // unique keys associated to collections in mixed
-                new_ref = Array::write(ref, alloc, out, out.only_modified, out.compress);
-            }
-            else {
-                // all the rest we don't want to compress it, at least for now (strings will be needed)
-                new_ref = Array::write(ref, alloc, out, out.only_modified, false);
+                case payload_idx_key:
+                    new_ref = Array::write(ref, alloc, out, out.only_modified, out.compress);
+                    break;
             }
         }
         written_leaf.set(i, new_ref);

src/realm/array_mixed.hpp

@@ -64,6 +64,15 @@ class ArrayMixed : public ArrayPayload, private Array {
     {
         Array::set_parent(parent, ndx_in_parent);
     }
+    bool need_string_interner() const override
+    {
+        return true;
+    }
+    virtual void set_string_interner(StringInterner* interner) const override
+    {
+        m_strings.set_string_interner(interner);
+    }
+
     void init_from_parent()
     {
         ref_type ref = get_ref_from_parent();
@@ -88,6 +97,7 @@ class ArrayMixed : public ArrayPayload, private Array {
     {
         return m_composite.get(ndx) == 0;
     }
+    std::optional<StringID> get_string_id(size_t ndx) const;
 
     void clear();
     void erase(size_t ndx);